chore: merge engine fixes, third pass

scripts/artifacts.py

@@ -208,11 +208,16 @@ def _strip_timestamps(text: str) -> str:
 class ArtifactLockBusy(RuntimeError):
     """Raised when another process already holds the artifact directory."""
 
-# A run that holds an artifact directory for its whole duration names it
-# here for the steps it spawns: they would otherwise be refused by it.
+# A run that holds artifact directories for its whole duration names them
+# here, os.pathsep apart, for the steps it spawns: they would otherwise be
+# refused by it. A pipeline holds the repository root and its --output-dir.
 HELD_LOCK_ENV = "RETROBIOS_HELD_ARTIFACT_DIR"
 
 
+def _held_directories() -> list[str]:
+    return [d for d in os.environ.get(HELD_LOCK_ENV, "").split(os.pathsep) if d]
+
+
 @contextlib.contextmanager
 def artifact_lock(directory: str, exclusive: bool = True) -> Iterator[None]:
     """Serialize access to a shared artifact directory across processes.
@@ -229,7 +234,7 @@ def artifact_lock(directory: str, exclusive: bool = True) -> Iterator[None]:
         return
 
     os.makedirs(directory, exist_ok=True)
-    if os.environ.get(HELD_LOCK_ENV) == os.path.realpath(directory):
+    if os.path.realpath(directory) in _held_directories():
         yield
         return
     lock_path = os.path.join(directory, ".lock")
@@ -256,16 +261,24 @@ def hold_artifact_lock(directory: str) -> Iterator[None]:
     run can purge the packs one step built before the next step checks
     them. The children this process spawns inherit the hold.
     """
-    with artifact_lock(directory):
-        previous = os.environ.get(HELD_LOCK_ENV)
-        os.environ[HELD_LOCK_ENV] = os.path.realpath(directory)
-        try:
-            yield
-        finally:
-            if previous is None:
-                os.environ.pop(HELD_LOCK_ENV, None)
-            else:
-                os.environ[HELD_LOCK_ENV] = previous
+    with artifact_lock(directory), publish_hold(directory):
+        yield
+
+
+@contextlib.contextmanager
+def publish_hold(directory: str) -> Iterator[None]:
+    """Name a directory this process holds to the steps it spawns."""
+    previous = os.environ.get(HELD_LOCK_ENV)
+    os.environ[HELD_LOCK_ENV] = os.pathsep.join(
+        [*_held_directories(), os.path.realpath(directory)]
+    )
+    try:
+        yield
+    finally:
+        if previous is None:
+            os.environ.pop(HELD_LOCK_ENV, None)
+        else:
+            os.environ[HELD_LOCK_ENV] = previous
 
 
 # What a pipeline run writes at fixed paths, whatever its --output-dir.
@@ -280,20 +293,40 @@ def hold_repository_outputs(root: str = ".") -> Iterator[None]:
     on dist/: both rewrote database.json, and the second to reach the
     manifest step found install/ held after minutes of work, or, when the
     steps did not overlap, rewrote the committed manifests from its own
-    state. The lock file is the repository root's .lock.
+    state. The lock file is the repository root's .lock. A pipeline passes
+    it to its steps with publish_hold.
     """
     stack = contextlib.ExitStack()
     try:
         stack.enter_context(artifact_lock(root))
     except ArtifactLockBusy as exc:
         raise ArtifactLockBusy(
-            f"another pipeline run is writing {', '.join(REPOSITORY_OUTPUTS)} "
+            f"another run is writing {', '.join(REPOSITORY_OUTPUTS)} "
             f"(lock: {os.path.join(root, '.lock')}). Wait for it to finish."
         ) from exc
     with stack:
         yield
 
 
+_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+
+def repository_output_lock(path: str) -> contextlib.AbstractContextManager[None]:
+    """The repository hold, when *path* is one of its fixed outputs.
+
+    Only a pipeline took it: generate_db.py run beside a pipeline replaced
+    database.json between its verify and its packs, and a manifest run on
+    install/ made the pipeline's own manifest step fail after minutes of
+    work. A path elsewhere (a scratch --output) takes nothing.
+    """
+    target = os.path.realpath(path)
+    for output in REPOSITORY_OUTPUTS:
+        fixed = os.path.join(_REPO_ROOT, output.rstrip("/"))
+        if target == fixed or target.startswith(fixed + os.sep):
+            return hold_repository_outputs(_REPO_ROOT)
+    return contextlib.nullcontext()
+
+
 def _build_timestamp(db: dict | None = None) -> str:
     """Timestamp for generated artifacts.
 

scripts/check_buildbot_system.py

@@ -15,13 +15,14 @@ from __future__ import annotations
 import argparse
 import json
 import logging
+import os
 import subprocess
 import sys
 import urllib.error
 import urllib.parse
 import urllib.request
 from pathlib import Path
-from common import yaml_load
+from common import InputMissing, yaml_load
 
 log = logging.getLogger(__name__)
 
@@ -194,6 +195,9 @@ def main() -> None:
         help="the data directory registry to check (default: %(default)s)",
     )
     args = parser.parse_args()
+    # Read after the buildbot answered, a mistyped path ended on a trace.
+    if not os.path.isfile(args.registry):
+        raise InputMissing(f"Registry not found: {args.registry}")
 
     report = check(args.registry)
 
@@ -209,10 +213,12 @@ def main() -> None:
             sys.exit(1)
 
     # Unreachable upstream means the freshness question was not answered, and
-    # a zero exit says it was answered "fresh". A new asset nobody tracks, or
-    # a tracked one the buildbot dropped, is a change to act on: a zero exit
-    # there let the online pipeline call the system directory current.
-    if report.get("error") or pending(report):
+    # a zero exit says it was answered "fresh". An asset whose HEAD went
+    # unanswered is the same question left open, as check_freshness counts
+    # UNKNOWN. A new asset nobody tracks, or a tracked one the buildbot
+    # dropped, is a change to act on: a zero exit there let the online
+    # pipeline call the system directory current.
+    if report.get("error") or pending(report) or unanswered(report):
         raise SystemExit(1)
 
 
@@ -221,6 +227,11 @@ def pending(report: dict) -> list[dict]:
     return [e for e in report.get("entries", []) if e["status"] in ("NEW", "STALE")]
 
 
+def unanswered(report: dict) -> list[dict]:
+    """Tracked entries whose remote version could not be read."""
+    return [e for e in report.get("entries", []) if e["status"] == "UNKNOWN"]
+
+
 if __name__ == "__main__":
     from common import run_cli
 

scripts/check_freshness.py

@@ -58,6 +58,7 @@ from common import (  # noqa: E402
     list_registered_platforms,
     load_emulator_profiles,
     load_platform_registry,
+    require_directory,
     upstream_profile_index,
     write_text_atomic,
     yaml_load,
@@ -1789,6 +1790,7 @@ def main() -> int:
     if unknown:
         parser.error(unknown_areas_message(unknown))
 
+    require_directory(args.platforms_dir, "--platforms-dir")
     platforms_dir = Path(args.platforms_dir)
     profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
     findings: list[Finding] = []

scripts/check_release.py

@@ -33,7 +33,7 @@ from pathlib import Path
 sys.path.insert(0, os.path.dirname(__file__))
 from build_port import ARCHIVES
 from check_release_assets import digests_of
-from common import list_release_pack_platforms
+from common import list_release_pack_platforms, require_directory
 from largefiles import LARGE_FILES_REPO
 from release_record import RECORD, load_record
 
@@ -208,6 +208,7 @@ def main() -> int:
     )
     parser.add_argument("--json", action="store_true", help="JSON output")
     args = parser.parse_args()
+    require_directory(args.platforms_dir, "--platforms-dir")
 
     record = load_record(args.record)
     tag = args.tag or record.get("tag")

scripts/common.py

@@ -15,6 +15,7 @@ import functools
 import json
 import os
 import re
+import signal
 import sys
 import zipfile
 from pathlib import Path
@@ -43,7 +44,12 @@ def require_directory(path: str, flag: str = "--bios-dir") -> None:
         raise SystemExit(f"ERROR: {flag} {path} is not a directory")
 
 
-class DatabaseUnreadable(FileNotFoundError):
+class InputMissing(FileNotFoundError):
+    """An input a command was pointed at does not exist. run_cli prints the
+    message on one line instead of a stack trace."""
+
+
+class DatabaseUnreadable(InputMissing):
     """The database a command was pointed at does not exist or is not JSON."""
 
 
@@ -503,7 +509,7 @@ def list_registered_platforms(
     """
     registry_path = os.path.join(platforms_dir, "_registry.yml")
     if not os.path.exists(registry_path):
-        raise FileNotFoundError(f"no platform registry at {registry_path}")
+        raise InputMissing(f"no platform registry at {registry_path}")
     with open(registry_path) as f:
         registry = yaml_load(f) or {}
     platforms = []
@@ -1214,6 +1220,19 @@ def archive_entry(archive: str, files: list[dict]) -> dict:
     return {**own, "members": members} if members else own
 
 
+def archive_is_optional(entry: dict, archive: str, files: list[dict]) -> bool:
+    """Optional as a whole: its own entry and every member it carries.
+
+    A --required-only pack leaves it out, and the report of that pack too.
+    geolith's _Required pack held neocd.zip and irrmaze.zip all the same.
+    """
+    return entry.get("required") is False and all(
+        member.get("required") is False
+        for member in files
+        if member.get("archive") == archive
+    )
+
+
 def holds_required_members(path: str, members: list[dict]) -> bool:
     """Whether an archive holds every member a profile requires inside it."""
     crcs = zip_member_crcs(path)
@@ -1317,7 +1336,7 @@ def _by_affinity(paths: list[str], file_entry: dict, dest_hint: str) -> list[str
     owners = _owner_tokens(file_entry)
     sized = any(file_entry.get(k) for k in ("size", "min_size", "max_size"))
 
-    def score(path: str) -> tuple[int, bool, bool, int]:
+    def score(path: str) -> tuple[int, bool, bool, bool, int]:
         fits = 0
         if sized and os.path.exists(path):
             fits = int(size_fits(file_entry, os.path.getsize(path)))
@@ -2181,12 +2200,11 @@ def load_emulator_profiles(
         return {}
     profiles = {}
     emu_path = Path(emulators_dir)
-    if not emu_path.exists():
-        print(
-            f"warning: no emulator profile directory at {emulators_dir}",
-            file=sys.stderr,
-        )
-        return profiles
+    if not emu_path.is_dir():
+        # Read as "no emulator", a mistyped --emulators-dir built a pack
+        # without any core file marked OK, and sitesources wrote {} over
+        # source-paths.json, both exiting 0.
+        raise InputMissing(f"no emulator profile directory at {emulators_dir}")
     for f in source_yaml_paths(emu_path):
         with open(f) as fh:
             profile = yaml_load(fh) or {}
@@ -3542,6 +3560,8 @@ from artifacts import (  # noqa: E402,F401
     artifact_lock,
     hold_artifact_lock,
     hold_repository_outputs,
+    publish_hold,
+    repository_output_lock,
     _TIMESTAMP_PATTERNS,
     _strip_timestamps,
 )
@@ -3560,6 +3580,10 @@ from dumpcatalog import (  # noqa: E402,F401
 )
 
 
+def _exit_on_signal(signum: int, _frame: object) -> None:
+    sys.exit(128 + signum)
+
+
 def run_cli(main: Callable[[], int | None]) -> None:
     """Run a command-line entry point to its exit status.
 
@@ -3567,11 +3591,19 @@ def run_cli(main: Callable[[], int | None]) -> None:
     and the next write died on a BrokenPipeError traceback. Output nobody
     reads any more ends the run quietly, with the status a shell gives a
     writer whose reader left.
+
+    SIGTERM (timeout, kill) and SIGHUP (a closed terminal) end the process
+    without unwinding by default: a pack build left its multi-GB hidden
+    .partial behind. Raised as SystemExit, they run the cleanup a failed
+    write runs, and the status stays the one a shell gives the signal.
     """
+    for name in ("SIGTERM", "SIGHUP"):
+        if hasattr(signal, name):
+            signal.signal(getattr(signal, name), _exit_on_signal)
     try:
         code = main()
         sys.stdout.flush()
-    except DatabaseUnreadable as exc:
+    except (InputMissing, ArtifactLockBusy) as exc:
         print(f"ERROR: {exc}", file=sys.stderr)
         sys.exit(1)
     except BrokenPipeError:

scripts/fileless_audit.py

@@ -25,7 +25,7 @@ from pathlib import Path
 sys.path.insert(0, os.path.dirname(__file__))
 
 import upstream
-from common import suggest_names
+from common import require_directory, suggest_names
 from profile_sync import collect_citations, ref_role, select_views, split_source_ref
 from safeparse import yaml_load
 
@@ -122,6 +122,7 @@ def main() -> None:
     args = parser.parse_args()
     if args.all == bool(args.emulators):
         parser.error("name profiles or pass --all, not both")
+    require_directory(args.emulators_dir, "--emulators-dir")
 
     known = sorted(path.stem for path in Path(args.emulators_dir).glob("*.yml"))
     unknown = [name for name in args.emulators if name not in known]

scripts/generate_db.py

@@ -26,6 +26,7 @@ from common import (
     DEFAULT_PROVENANCE_DIR,
     list_registered_platforms,
     load_provenance_snapshots,
+    repository_output_lock,
     require_yaml,
     set_plain_mode,
     source_yaml_paths,
@@ -430,7 +431,11 @@ def main() -> int:
         help="Directory with dump-catalog snapshots",
     )
     args = parser.parse_args()
+    with repository_output_lock(args.output):
+        return _generate(args)
 
+
+def _generate(args: argparse.Namespace) -> int:
     bios_dir = Path(args.bios_dir)
     if not bios_dir.is_dir():
         print(f"Error: BIOS directory '{bios_dir}' not found", file=sys.stderr)
@@ -742,4 +747,6 @@ def _collect_all_aliases(
 
 
 if __name__ == "__main__":
-    sys.exit(main() or 0)
+    from common import run_cli
+
+    run_cli(main)

scripts/generate_games_index.py

@@ -49,6 +49,7 @@ from common import (  # noqa: E402
     load_emulator_profiles,
     load_platform_config,
     native_layer,
+    repository_output_lock,
     run_cli,
 )
 from region import normalize_declared  # noqa: E402
@@ -661,7 +662,13 @@ def main() -> int:
                         help="exit 1 when a file would change, naming what differs (implies --quiet)")
     parser.add_argument("--quiet", action="store_true", help="print only failures")
     args = parser.parse_args()
+    if args.check:
+        return _index(args)
+    with repository_output_lock(str(args.output_dir)):
+        return _index(args)
 
+
+def _index(args: argparse.Namespace) -> int:
     files, coverage = render(args.output_dir)
     if not (args.quiet or args.check):
         print_coverage(coverage, files)

scripts/generate_pack.py

@@ -12,6 +12,13 @@ and 3-tier storage (embedded/external/user_provided).
 
 from __future__ import annotations
 
+import sys
+
+# The README's first command. Under 3.8 it stopped on a SyntaxError that read
+# like a damaged file. The build needs 3.9, as pipeline.py says.
+if sys.version_info < (3, 9):
+    sys.exit(f"generate_pack.py needs Python 3.9 or newer, this is {sys.version.split()[0]}")
+
 from collections.abc import Callable, Iterable, Iterator
 
 from artifacts import _build_timestamp, set_plain_mode
@@ -23,7 +30,6 @@ import json
 import os
 import re
 import shutil
-import sys
 import tempfile
 import zipfile
 from pathlib import Path
@@ -41,7 +47,9 @@ from common import (
     archive_entry,
     profiles_for_systems,
     write_text_atomic,
+    archive_is_optional,
     artifact_lock,
+    repository_output_lock,
     ArtifactLockBusy,
     build_target_cores_cache,
     build_zip_contents_index,
@@ -602,6 +610,7 @@ def _report_pack_counts(
     core_count: int,
     source: str,
     verification_mode: str,
+    tree_count: int = 0,
 ) -> None:
     """Print the line the pipeline's consistency check reads.
 
@@ -633,10 +642,11 @@ def _report_pack_counts(
             f"{core_count} from emulator profiles [{verification_mode}]"
         )
     else:
-        baseline = total_files - core_count
+        baseline = total_files - core_count - tree_count
+        trees = f"{tree_count} in core data trees + " if tree_count else ""
         print(
             f"  {zip_path}: {total_files} files packed ({baseline} baseline + "
-            f"{core_count} from cores), {', '.join(parts)} [{verification_mode}]"
+            f"{trees}{core_count} from cores), {', '.join(parts)} [{verification_mode}]"
         )
 
 def _select_variants(
@@ -1308,11 +1318,19 @@ def generate_pack(
           core_trees = [t for t in core_trees if not _extras_for_systems([t], owned)]
       elif system_filter:
           core_trees = _extras_for_systems(core_trees, system_filter)
+      # Counted apart: the trees a core reads are not on the platform's
+      # list. MyMinUI declares nothing and its line read "6944 baseline".
       total_files += _pack_data_directories(
           zf, pack_systems, data_registry, platform_name, base_dest,
           flatten, case_insensitive, seen_destinations, seen_lower,
+          seen_parents, required_only=required_only,
+      )
+      tree_count = _pack_data_directories(
+          zf, {}, data_registry, platform_name, base_dest,
+          flatten, case_insensitive, seen_destinations, seen_lower,
           seen_parents, required_only=required_only, core_trees=core_trees,
       )
+      total_files += tree_count
 
       # README.txt for users: personalized step-by-step per platform
       num_systems = len(pack_systems)
@@ -1351,7 +1369,7 @@ def generate_pack(
 
     _report_pack_counts(
         zip_path, file_status, total_files, core_count, source,
-        verification_mode,
+        verification_mode, tree_count=tree_count,
     )
     if cut is not None:
         _report_left_out(
@@ -1456,18 +1474,6 @@ class DamagedArchiveError(OSError):
 
 
 
-def _archive_is_optional(archive_entry: dict, archive_name: str, files: list[dict]) -> bool:
-    """Optional as a whole: its own entry and every member it carries.
-
-    geolith's _Required pack held neocd.zip and irrmaze.zip all the same.
-    """
-    return archive_entry.get("required") is False and all(
-        member.get("required") is False
-        for member in files
-        if member.get("archive") == archive_name
-    )
-
-
 def _pack_data_tree(
     zf: zipfile.ZipFile,
     local_cache: str,
@@ -1656,7 +1662,7 @@ def generate_emulator_pack(
                 )
 
                 own_entry = archive_entry(archive_name, files)
-                if required_only and _archive_is_optional(own_entry, archive_name, files):
+                if required_only and archive_is_optional(own_entry, archive_name, files):
                     continue
                 local_path, status = resolve_file(
                     own_entry,
@@ -2803,9 +2809,11 @@ def _run_manifest_mode(
 
 @contextlib.contextmanager
 def _pack_output_lock(output_dir: str, exclusive: bool = True) -> Iterator[None]:
-    """Hold the output directory for the duration of a pack run."""
+    """Hold the output directory for the duration of a pack run, and the
+    repository's fixed outputs when it is one of them (install/)."""
     try:
-        with artifact_lock(output_dir, exclusive=exclusive):
+        with artifact_lock(output_dir, exclusive=exclusive), \
+                repository_output_lock(output_dir) if exclusive else contextlib.nullcontext():
             yield
     except ArtifactLockBusy as exc:
         print(f"ERROR: {exc}", file=sys.stderr)
@@ -3031,6 +3039,8 @@ def _run_platform_packs(
     # Per pack, the systems each of its files is for, written into its
     # manifest.json once the pack is verified (pack_systems).
     systems_for: dict[str, dict] = {}
+    # The packs this run writes, the only ones it verifies and rewrites.
+    built: set[str] = set()
 
     failed: list[str] = []
     for source, required_only in variants:
@@ -3131,7 +3141,12 @@ def _run_platform_packs(
                 # the output directory already held and exit 0.
                 if not _produced(args.split, zip_path, zip_paths):
                     failed.append(representative)
-                elif zip_path and not system_filter:
+                    continue
+                built.update(
+                    os.path.relpath(path, args.output_dir)
+                    for path in ([zip_path] if zip_path else zip_paths)
+                )
+                if zip_path and not system_filter:
                     # The install list of the same choices names each file's
                     # systems: the pack records them, so a pack alone (a
                     # handheld without Wi-Fi) can be narrowed by system.
@@ -3154,6 +3169,7 @@ def _run_platform_packs(
         db,
         platforms_dir=args.platforms_dir,
         emulators_dir=args.emulators_dir,
+        built=built,
         pack_systems_by_name=systems_for,
         skip_conformance=skip_conf,
         data_registry=data_registry,
@@ -3170,6 +3186,10 @@ def _run_platform_packs(
                     db,
                     platforms_dir=args.platforms_dir,
                     emulators_dir=args.emulators_dir,
+                    built={
+                        os.path.basename(name) for name in built
+                        if os.path.dirname(name) == entry
+                    },
                     skip_conformance=True,
                     data_registry=data_registry,
                     regions=getattr(args, "regions", None),
@@ -3567,6 +3587,9 @@ def _main(held: contextlib.ExitStack) -> None:
         except ValueError as exc:
             print(f"ERROR: {exc}", file=sys.stderr)
             sys.exit(1)
+        except OSError as exc:
+            print(f"ERROR: --from-md5-file {args.from_md5_file}: {exc.strerror}", file=sys.stderr)
+            sys.exit(1)
         if not hashes:
             print(
                 f"ERROR: no hash in the input. Accepted: {ACCEPTED_HASHES}",
@@ -5089,10 +5112,8 @@ def inject_manifest(zip_path: str, manifest: dict) -> None:
             with zipfile.ZipFile(scratch, "a", zipfile.ZIP_DEFLATED) as zf:
                 _write_generated_member(zf, "manifest.json", manifest_json)
         else:
-            with (
-                zipfile.ZipFile(zip_path, "r") as src,
-                zipfile.ZipFile(scratch, "w", zipfile.ZIP_DEFLATED) as dst,
-            ):
+            with zipfile.ZipFile(zip_path, "r") as src, \
+                    zipfile.ZipFile(scratch, "w", zipfile.ZIP_DEFLATED) as dst:
                 for item in src.infolist():
                     if item.filename == "manifest.json":
                         continue
@@ -5287,12 +5308,18 @@ def verify_and_finalize_packs(
     emulators_dir: str = "emulators",
     no_game_data: bool = False,
     pack_systems_by_name: dict[str, dict] | None = None,
+    built: Iterable[str] | None = None,
 ) -> bool:
     """Verify all packs, inject manifests, generate SHA256SUMS.
 
     pack_systems_by_name gives, per pack file name, the systems its
     manifest.json records (pack_systems).
 
+    ``built`` names the packs this run wrote, and only those are verified
+    and get a manifest. A pack another run left in the directory was
+    reopened, recompressed and given a manifest without its pack_systems,
+    which made the installer refuse to narrow it. None takes every pack.
+
     Two-stage verification:
     1. Hash check against database.json (integrity)
     2. Extract + verify against platform config (conformance)
@@ -5309,14 +5336,17 @@ def verify_and_finalize_packs(
     expected = _expected_pack_names(
         registered, platforms_dir, regions, target_name, no_game_data=no_game_data
     )
+    packs = _pack_archives(output_dir)
+    if built is not None:
+        packs = [name for name in packs if name in set(built)]
     pack_to_platform: dict[str, list[str]] = {}
-    for name in _pack_archives(output_dir):
+    for name in packs:
         for pname in registered:
             if name in expected[pname]:
                 pack_to_platform.setdefault(name, []).append(pname)
 
     failing: list[str] = []
-    for name in _pack_archives(output_dir):
+    for name in packs:
         zip_path = os.path.join(output_dir, name)
 
         # Stage 1: database integrity

scripts/generate_readme.py

@@ -20,6 +20,7 @@ from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
 from common import (
+    InputMissing,
     compute_composition,
     count_catalog_matched,
     declared_root,
@@ -29,6 +30,7 @@ from common import (
     load_emulator_profiles,
     load_platform_config,
     load_platform_registry,
+    repository_output_lock,
     resolve_platform_cores,
     system_index,
     unique_emulator_profiles,
@@ -211,16 +213,23 @@ def platform_coverages(
                 coverages[name] = compute_coverage(
                     name, platforms_dir, db, data_registry, emulators_dir=emulators_dir,
                 )
+            except InputMissing:
+                # A missing profile or platform directory is no platform
+                # left out: it emptied every one and cached the result.
+                raise
             except FileNotFoundError as exc:
                 print(f"  {name}: skipped ({exc})", file=sys.stderr)
         COVERAGE_CACHE_DIR.mkdir(parents=True, exist_ok=True)
-        for stale in COVERAGE_CACHE_DIR.glob("*.pickle"):
-            stale.unlink()
         scratch = cache.with_suffix(f".{os.getpid()}.tmp")
         with scratch.open("wb") as handle:
             pickle.dump({"platforms": names, "coverages": coverages}, handle,
                         protocol=pickle.HIGHEST_PROTOCOL)
         scratch.replace(cache)
+        # generate_site and generate_readme both purge here. One unlinking
+        # what the other had just listed ended it on FileNotFoundError.
+        for stale in COVERAGE_CACHE_DIR.glob("*.pickle"):
+            if stale != cache:
+                stale.unlink(missing_ok=True)
     ordered = {name: coverages[name] for name in platform_names if name in coverages}
     if report:
         for name in names:
@@ -974,7 +983,7 @@ validation above is its check. For code, profiles or platform files, these are
 the checks CI runs on a pull request (Python 3.12 or newer):
 
 ```bash
-pip install pyyaml "jsonschema[format-nongpl]" markdown pyte hypothesis vermin
+pip install pyyaml "jsonschema[format-nongpl]" markdown pyte hypothesis vermin setuptools
 python scripts/validate_pr.py --changed          # the BIOS files the branch adds
 python scripts/validate_schemas.py --source-only # profiles and platform files
 python scripts/profile_sync.py --added --base upstream/main --strict  # each profile the branch adds
@@ -1036,9 +1045,13 @@ def main() -> None:
 
     # Coverage resolves against data/: a refresh swapping a tree mid-read
     # published counts taken from half of it.
-    with contextlib.ExitStack() as held:
+    with repository_output_lock("README.md"), contextlib.ExitStack() as held:
         hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
         readme = generate_readme(db, args.platforms_dir, args.db)
+        _write(readme)
+
+
+def _write(readme: str) -> None:
     status = "Generated" if write_if_changed("README.md", readme) else "Unchanged"
     print(f"{status} ./README.md")
 

scripts/generate_site.py

@@ -12080,11 +12080,14 @@ def main() -> None:
         # files in the sweep, so the second is refused instead.
         import contextlib
 
-        from common import load_data_dir_registry
+        from common import load_data_dir_registry, repository_output_lock
         from refresh_data_dirs import hold_data_dirs
 
         # The coverage pages resolve against data/, held while they are read.
-        with artifact_lock(args.docs_dir), contextlib.ExitStack() as held:
+        # Whatever --docs-dir says, website/src/data and wiki/ are written.
+        with artifact_lock(args.docs_dir), \
+                repository_output_lock(str(PUBLISHED_PARTS_FILE)), \
+                contextlib.ExitStack() as held:
             hold_data_dirs(held, load_data_dir_registry(args.platforms_dir))
             _generate(args)
     except ArtifactLockBusy as exc:

scripts/generate_truth.py

@@ -21,6 +21,7 @@ from common import (
     load_emulator_profiles,
     load_platform_config,
     load_target_config,
+    require_directory,
     require_platform,
     require_yaml,
     truth_target_dir,
@@ -72,6 +73,7 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
 
 def main(argv: list[str] | None = None) -> None:
     args = parse_args(argv)
+    require_directory(args.platforms_dir, "--platforms-dir")
 
     registry_path = os.path.join(args.platforms_dir, "_registry.yml")
     with open(registry_path) as f:

scripts/mame_ref_audit.py

@@ -27,6 +27,7 @@ from dataclasses import dataclass
 sys.path.insert(0, os.path.dirname(__file__))
 
 import upstream
+from common import require_directory
 from safeparse import yaml_load
 from scraper.mame_parser import _MACHINE_MACROS, strip_comments
 
@@ -145,6 +146,7 @@ def main() -> None:
         help="rewrite the refs the audit can settle",
     )
     args = parser.parse_args()
+    require_directory(args.emulators_dir, "--emulators-dir")
 
     total = 0
     for name in args.emulators:

scripts/packextras.py

@@ -1552,11 +1552,19 @@ def extras_for_systems(extras: list[dict], system_ids: list[str]) -> list[dict]:
 
 
 def report_owned_by_systems(report: list[dict], system_ids: list[str]) -> list[dict]:
-    """Gap report entries whose extra a pack built with --system carries."""
+    """Gap report entries whose extra a pack built with --system carries.
+
+    The other cores whose claim rides on the entry count, as the builder
+    counts them: MAME 2009's taitofx1.zip went into a --system
+    panasonic-3do pack for MAME 2016 and was missing from the report.
+    """
+    def owner(entry: dict) -> dict:
+        return {"source_system": entry.get("system"), "source_systems": entry.get("systems", [])}
+
     return [
         u for u in report
         if extras_for_systems(
-            [{"source_system": u.get("system"), "source_systems": u.get("systems", [])}],
+            [{**owner(u), "other_claims": [owner(c) for c in u.get("other_claims", [])]}],
             system_ids,
         )
     ]

scripts/pipeline.py

@@ -57,8 +57,8 @@ import tempfile
 import time
 from pathlib import Path
 
-# generate_pack's parenthesized context managers do not compile before
-# CPython 3.9, which builds the same database, manifests and packs as 3.12.
+# The build scripts need CPython 3.9, which builds the same database,
+# manifests and packs as 3.12.
 # On an older interpreter the run got through the first steps and failed in
 # the packs, so it stops before step 0. A test compiles every script under
 # this floor.
@@ -73,6 +73,7 @@ from common import (
     hold_artifact_lock,
     hold_repository_outputs,
     load_data_dir_registry,
+    publish_hold,
 )
 from refresh_data_dirs import hold_data_dirs
 
@@ -522,15 +523,43 @@ def check_pack_destinations(
     return all_ok
 
 
+# The narrowings an emulator or system pack takes, compared by
+# check_emulator_packs. A test holds this to the builder's parameters.
+EMULATOR_NARROWINGS_COMPARED = frozenset({"standalone", "regions", "required_only"})
+
+
+def emulator_variants(profiles: list[dict], full: bool) -> list[tuple[str, dict]]:
+    """(label, builder arguments) of each narrowed emulator pack compared.
+
+    --required-only always, and each region the profiles' files declare,
+    sampled as check_narrowed_packs samples them unless *full*. A profile
+    without a regional file has nothing --region could withdraw.
+    """
+    import region as region_mod  # noqa: PLC0415
+
+    regions = sorted({
+        r for profile in profiles for fe in profile.get("files") or []
+        for r in region_mod.normalize_declared(fe.get("region"))
+    })
+    variants: list[tuple[str, dict]] = [("", {}), (" --required-only", {"required_only": True})]
+    variants += [
+        (f" --region {regions[i]}", {"regions": [regions[i]]})
+        for i in _matrix_coverage(full, len(regions))
+    ]
+    return variants
+
+
 def check_emulator_packs(
     emulators_dir: str = "emulators", platforms_dir: str = "platforms",
-    db_path: str = "database.json",
+    db_path: str = "database.json", full: bool = False,
 ) -> bool:
     """Build every emulator and system pack in memory and compare it with verify.
 
-    Each profile in each build it has (libretro, standalone): the paths the
-    pack fills with the profile's files against the ones verify credits. No
-    archive is written and no file read, so all of them take seconds.
+    Each profile in each build it has (libretro, standalone), full and under
+    each narrowing of emulator_variants: the paths the pack fills with the
+    profile's files against the ones verify credits. The region and
+    required-only rules are written twice, once per side, and only this
+    comparison holds them together. No archive is written and no file read.
     """
     import generate_pack  # noqa: PLC0415
     import verify  # noqa: PLC0415
@@ -548,6 +577,26 @@ def check_emulator_packs(
     db = load_database(db_path)
     zip_contents = build_zip_contents_index(db)
     profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
+
+    def compare(names: list[str], standalone: bool, label: str, **narrowing: object) -> bool:
+        record = generate_pack.PackRecord()
+        with contextlib.redirect_stdout(io.StringIO()), \
+                contextlib.redirect_stderr(io.StringIO()):
+            generate_pack.generate_emulator_pack(
+                names, emulators_dir, db, "bios", "", standalone, zip_contents,
+                offline=True, platforms_dir=platforms_dir, record=record, **narrowing,
+            )
+            report = verify.verify_emulator(
+                names, emulators_dir, db, standalone, platforms_dir=platforms_dir,
+                **narrowing,
+            )
+        lines = _set_difference_lines(
+            label, set(report["pack_destinations"]), set(record.files)
+        )
+        if lines:
+            print("\n".join(lines))
+        return not lines
+
     all_ok = True
     compared = 0
     for name in sorted(profiles):
@@ -558,23 +607,10 @@ def check_emulator_packs(
                 select_emulator_profiles([name], profiles, standalone)
             except ProfileSelectionError:
                 continue
-            record = generate_pack.PackRecord()
-            with contextlib.redirect_stdout(io.StringIO()):
-                generate_pack.generate_emulator_pack(
-                    [name], emulators_dir, db, "bios", "", standalone, zip_contents,
-                    offline=True, platforms_dir=platforms_dir, record=record,
-                )
-                report = verify.verify_emulator(
-                    [name], emulators_dir, db, standalone, platforms_dir=platforms_dir,
-                )
-            compared += 1
-            label = f"{name}{' --standalone' if standalone else ''}"
-            lines = _set_difference_lines(
-                label, set(report["pack_destinations"]), set(record.files)
-            )
-            if lines:
-                print("\n".join(lines))
-                all_ok = False
+            for tag, narrowing in emulator_variants([profiles[name]], full):
+                compared += 1
+                label = f"{name}{' --standalone' if standalone else ''}{tag}"
+                all_ok = compare([name], standalone, label, **narrowing) and all_ok
     print(f"  {compared} emulator packs compared, {'all equal' if all_ok else 'differences above'}")
     # A system pack builds every profile of the system together: a placement
     # rule that only goes wrong when two profiles share a path (the bluemsx
@@ -592,34 +628,24 @@ def check_emulator_packs(
             matching = profiles_for_systems(profiles, [sid], standalone)
             if len(matching) < 2:
                 continue
-            record = generate_pack.PackRecord()
-            with contextlib.redirect_stdout(io.StringIO()), \
-                    contextlib.redirect_stderr(io.StringIO()):
-                generate_pack.generate_emulator_pack(
-                    matching, emulators_dir, db, "bios", "", standalone, zip_contents,
-                    offline=True, platforms_dir=platforms_dir, record=record,
-                    system_ids=[sid],
-                )
-                report = verify.verify_emulator(
-                    matching, emulators_dir, db, standalone, platforms_dir=platforms_dir,
-                    system_ids=[sid],
-                )
-            compared += 1
-            label = f"--system {sid}{' --standalone' if standalone else ''}"
-            lines = _set_difference_lines(
-                label, set(report["pack_destinations"]), set(record.files)
-            )
-            if lines:
-                print("\n".join(lines))
-                systems_ok = False
+            for tag, narrowing in emulator_variants(
+                [profiles[n] for n in matching if n in profiles], full
+            ):
+                compared += 1
+                label = f"--system {sid}{' --standalone' if standalone else ''}{tag}"
+                systems_ok = compare(
+                    matching, standalone, label, system_ids=[sid], **narrowing
+                ) and systems_ok
     print(f"  {compared} system packs compared, "
           f"{'all equal' if systems_ok else 'differences above'}")
     return all_ok and systems_ok
 
 
-# How each _narrowings() dimension is compared by check_narrowed_packs, or why
-# it is not. A test reads the builder's signature against these two tables, so
-# a new narrowing fails it until it is compared or a reason is written here.
+# How each _narrowings() dimension of a platform pack is compared by
+# check_narrowed_packs, or why it is not. A test reads the builder's
+# signature against these two tables, so a new narrowing fails it until it
+# is compared or a reason is written here. Emulator packs have their own
+# table, EMULATOR_NARROWINGS_COMPARED.
 NARROWINGS_COMPARED = frozenset({"regions", "target_name", "required_only", "no_game_data"})
 NARROWINGS_NOT_COMPARED = {
     "standalone": "an emulator pack mode, compared by check_emulator_packs",
@@ -790,6 +816,25 @@ class _Skipped:
 SKIPPED = _Skipped()
 
 
+def integrity_command(args: argparse.Namespace) -> list[str]:
+    """Step 6: check the packs step 4 built, under the same expectation."""
+    cmd = [
+        sys.executable, "scripts/generate_pack.py", "--all", "--verify-packs",
+        "--output-dir", args.output_dir,
+    ]
+    if args.include_archived:
+        cmd.append("--include-archived")
+    # Without the target a targeted run reported every system the target
+    # removed as missing from a pack that was exactly right. Without
+    # --all-variants the five narrowed packs were never opened, and an empty
+    # _Platform pack passed.
+    if args.target:
+        cmd.extend(["--target", args.target])
+    if args.all_variants:
+        cmd.append("--all-variants")
+    return cmd
+
+
 def _refuse_unapplied_flags(
     parser: argparse.ArgumentParser, args: argparse.Namespace
 ) -> None:
@@ -797,7 +842,7 @@ def _refuse_unapplied_flags(
 
     --all-variants builds every source itself, so a --source beside it is
     dropped; --skip-packs builds none, so both are, and step 5 does not run
-    for --full-narrowings to widen.
+    for --full-narrowings to widen, nor does it without a full pack.
     """
     # --source defaults to None so a typed `--source full` is seen: compared
     # with its default value, it went through beside the modes that refuse it.
@@ -810,6 +855,10 @@ def _refuse_unapplied_flags(
                          ("--full-narrowings", args.full_narrowings)):
             if on:
                 parser.error(f"{flag} is incompatible with --skip-packs")
+    # Step 5 compares the full pack, and --source platform or truth builds
+    # none: the comparison is skipped and the flag had no effect.
+    if args.full_narrowings and source_given and args.source != "full":
+        parser.error(f"--full-narrowings is incompatible with --source {args.source}")
     if args.source is None:
         args.source = "full"
 
@@ -940,7 +989,8 @@ def main() -> None:
     parser.add_argument(
         "--full-narrowings", action="store_true",
         help="Step 5 compares every region, target and system of each "
-        "platform, not a sample of three (about 25 minutes)",
+        "platform, and every region of each emulator and system pack, not a "
+        "sample of three (about 25 minutes)",
     )
     parser.add_argument(
         "--check-buildbot",
@@ -979,9 +1029,11 @@ def main() -> None:
     # rewrote them beside the run on dist/.
     held = contextlib.ExitStack()
     try:
-        held.enter_context(
-            hold_repository_outputs(str(Path(__file__).resolve().parent.parent))
-        )
+        root = str(Path(__file__).resolve().parent.parent)
+        held.enter_context(hold_repository_outputs(root))
+        # Its steps write database.json, install/ and README.md themselves,
+        # and take the same lock to do so.
+        held.enter_context(publish_hold(root))
         if not args.skip_packs:
             held.enter_context(hold_artifact_lock(args.output_dir))
     except ArtifactLockBusy as exc:
@@ -1309,7 +1361,7 @@ def main() -> None:
         )
         # Counts can agree while the paths differ, both ways at once.
         ok = check_pack_destinations(destinations_file, pack_output) and ok
-        ok = check_emulator_packs() and ok
+        ok = check_emulator_packs(full=args.full_narrowings) and ok
         ok = check_narrowed_packs(
             include_archived=args.include_archived, full=args.full_narrowings,
         ) and ok
@@ -1330,22 +1382,7 @@ def main() -> None:
         skip_notice("pack_integrity", "no full pack built")
         results["pack_integrity"] = SKIPPED
     elif not args.skip_packs:
-        integrity_cmd = [
-            sys.executable,
-            "scripts/generate_pack.py",
-            "--all",
-            "--verify-packs",
-            "--output-dir",
-            args.output_dir,
-        ]
-        if args.include_archived:
-            integrity_cmd.append("--include-archived")
-        # Step 4 built with the target, so step 6 has to check against the
-        # same expectation. Without it a targeted run reported every system
-        # the target removed as missing from a pack that was exactly right.
-        if args.target:
-            integrity_cmd.extend(["--target", args.target])
-        ok, _ = run(integrity_cmd, STEP_LABELS["pack_integrity"])
+        ok, _ = run(integrity_command(args), STEP_LABELS["pack_integrity"])
         results["pack_integrity"] = ok
         all_ok = all_ok and ok
     else:

scripts/refresh_data_dirs.py

@@ -27,7 +27,7 @@ import zipfile
 from collections.abc import Iterator
 from pathlib import Path
 from artifacts import file_lock as _file_lock, set_plain_mode
-from common import github_token, platform_arg, require_platform, suggest_names, yaml_load
+from common import InputMissing, github_token, platform_arg, require_platform, suggest_names, yaml_load
 
 try:
     import yaml
@@ -53,7 +53,7 @@ def load_registry(registry_path: str = DEFAULT_REGISTRY) -> dict[str, dict]:
         raise ImportError("PyYAML required: pip install pyyaml")
     path = Path(registry_path)
     if not path.exists():
-        raise FileNotFoundError(f"Registry not found: {registry_path}")
+        raise InputMissing(f"Registry not found: {registry_path}")
     with open(path) as f:
         data = yaml_load(f) or {}
     return data.get("data_directories", {})
@@ -442,13 +442,45 @@ def refresh_entry(
     The decision and the swap run under one lock per directory: a run that
     waited for another session re-reads the version it recorded and finds
     nothing left to do.
+
+    The exclusive lock waits for every reader, and a pipeline reads the
+    trees for its whole run: a second build sat on "waiting" for a tree the
+    remote check would have found current. The check is made first without
+    it, and only a tree to replace waits.
     """
     if dry_run:
         return _refresh_entry(key, entry, force, dry_run, versions_path)
+    if not force:
+        needs, _remote, cached_tag = _decide(key, entry, False, versions_path)
+        if needs is None:
+            return None
+        if not needs:
+            log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag))
+            return False
     with cache_lock(entry["local_cache"]):
         return _refresh_entry(key, entry, force, dry_run, versions_path)
 
 
+def _decide(
+    key: str, entry: dict, force: bool, versions_path: str
+) -> tuple[bool | None, str | None, str | None]:
+    """(whether the tree needs a refresh, the remote tag read, the recorded
+    tag). None when the remote could not be read."""
+    version = entry.get("version", "master")
+    cached = _load_versions(versions_path).get(key, {})
+    cached_tag = cached.get("sha") or cached.get("etag")
+    if force or not Path(entry["local_cache"]).exists():
+        return True, None, cached_tag
+    if entry.get("source_type", "tarball") == "zip":
+        remote_tag = _get_remote_etag(entry["source_url"].format(version=version))
+    else:
+        remote_tag = get_remote_sha(entry["source_url"], version)
+    if remote_tag is None:
+        log.warning("[%s] could not check remote, skipping", key)
+        return None, None, cached_tag
+    return remote_tag != cached_tag, remote_tag, cached_tag
+
+
 def _refresh_entry(
     key: str, entry: dict, force: bool, dry_run: bool, versions_path: str
 ) -> bool | None:
@@ -458,23 +490,9 @@ def _refresh_entry(
     local_cache = entry["local_cache"]
     exclude = entry.get("exclude", [])
 
-    versions = _load_versions(versions_path)
-    cached = versions.get(key, {})
-    cached_tag = cached.get("sha") or cached.get("etag")
-
-    needs_refresh = force or not Path(local_cache).exists()
-
-    remote_tag: str | None = None
-    if not needs_refresh:
-        if source_type == "zip":
-            remote_tag = _get_remote_etag(source_url)
-        else:
-            remote_tag = get_remote_sha(entry["source_url"], version)
-        if remote_tag is None:
-            log.warning("[%s] could not check remote, skipping", key)
-            return None
-        needs_refresh = remote_tag != cached_tag
-
+    needs_refresh, remote_tag, cached_tag = _decide(key, entry, force, versions_path)
+    if needs_refresh is None:
+        return None
     if not needs_refresh:
         log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag))
         return False

scripts/release_notes.py

@@ -26,6 +26,7 @@ from common import (
     list_release_pack_platforms,
     load_platform_config,
     load_platform_registry,
+    require_directory,
     write_text_atomic,
 )
 from siteguides import install_command
@@ -162,6 +163,7 @@ def main() -> int:
     parser.add_argument("--output", required=True, type=Path,
                         help="notes file to write, passed to gh release create --notes-file")
     args = parser.parse_args()
+    require_directory(args.platforms_dir, "--platforms-dir")
 
     record = release_record.load_record(args.record)
     if not record.get("packs"):

scripts/scraper/arcade_parent_importer.py

@@ -50,7 +50,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 
 from ..artifacts import write_text_atomic
-from ..common import load_emulator_profiles
+from ..common import load_emulator_profiles, run_cli
 from .romset_dat_importer import (
     LISTXML_CHUNK,
     MAME_LISTXML_URL,
@@ -1203,4 +1203,4 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    run_cli(main)

scripts/scraper/romset_dat_importer.py

@@ -45,6 +45,7 @@ from ..common import (
     list_registered_platforms,
     load_emulator_profiles,
     load_platform_config,
+    run_cli,
 )
 from ..dumpcatalog import build_snapshot, load_provenance_snapshots, write_snapshot
 from .logiqx_parser import LogiqxDat, parse_logiqx
@@ -665,4 +666,4 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    run_cli(main)

scripts/sitesources.py

@@ -735,13 +735,18 @@ def main() -> int:
     if not args.fetch and not any(path.is_dir() for path in cache_dirs):
         print(f"Error: no tree cache at {', '.join(map(str, cache_dirs))}", file=sys.stderr)
         return 1
-    from common import load_emulator_profiles, write_if_changed  # noqa: PLC0415
+    from common import (  # noqa: PLC0415
+        load_emulator_profiles,
+        repository_output_lock,
+        write_if_changed,
+    )
 
     profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
-    held, unread = build_source_paths(profiles, cache_dirs, args.fetch)
-    held = keep_unread(held, unread, read_record(args.output))
-    args.output.parent.mkdir(parents=True, exist_ok=True)
-    write_if_changed(str(args.output), json.dumps(held, indent=1, sort_keys=True) + "\n")
+    with repository_output_lock(str(args.output)):
+        held, unread = build_source_paths(profiles, cache_dirs, args.fetch)
+        held = keep_unread(held, unread, read_record(args.output))
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        write_if_changed(str(args.output), json.dumps(held, indent=1, sort_keys=True) + "\n")
     print(f"{sum(map(len, held.values()))} paths held in {len(held)} trees, "
           f"{len(unread)} trees not cached: {args.output}")
     return 0

scripts/validate_pr.py

@@ -34,6 +34,7 @@ from common import (
     load_emulator_profiles,
     load_platform_config,
     parse_md5_list,
+    require_directory,
 )
 from hashing import parse_hash_list
 from nativemode import digest_algorithm
@@ -806,6 +807,7 @@ def main() -> None:
         return
 
     db = _load_database_if_any(args.db)
+    require_directory(args.platforms_dir, "--platforms-dir")
     platform_hashes = load_platform_hashes(args.platforms_dir, db)
     profile_hashes = load_profile_hashes(args.emulators_dir, db)
 

scripts/validate_site.py

@@ -499,6 +499,9 @@ def main() -> None:
         help="stop listing after this many errors (default: %(default)s)",
     )
     args = parser.parse_args()
+    from common import require_directory  # noqa: PLC0415
+
+    require_directory(str(args.platforms_dir), "--platforms-dir")
 
     try:
         issues = validate_site(Path(args.site_dir), args.base, args.platforms_dir, args.record)

scripts/verify.py

@@ -43,6 +43,7 @@ from common import (
     suggest_names,
     require_platform,
     archive_entry,
+    archive_is_optional,
     archive_members,
     profiles_for_systems,
     PROFILE_IDENTITY_FIELDS,
@@ -2056,12 +2057,16 @@ def verify_emulator(
     regions: list[str] | None = None,
     platforms_dir: str = _REPO_PLATFORMS,
     system_ids: list[str] | None = None,
+    required_only: bool = False,
 ) -> dict:
     """Verify files for specific emulator profiles.
 
     A region priority list narrows the report the same way a pack built with the
     same list would be narrowed, through the emulator pack's own drop set.
     `system_ids` keeps the entries of those systems, as the system pack does.
+    `required_only` reports on the pack built with --required-only: a loose
+    file or tree marked ``required: false`` is left out, and an archive when
+    it and every member it carries are optional, as the builder leaves them.
     """
     from packextras import expand_agnostic_entries
     from packresolve import resolve_file
@@ -2105,7 +2110,7 @@ def verify_emulator(
     if regions:
         from packextras import emulator_region_drops
 
-        region_drops = emulator_region_drops(selected, standalone, regions, db=db)
+        region_drops = emulator_region_drops(selected, standalone, regions, required_only, db=db)
 
     # The emulator pack walks profiles in name order and keeps the first
     # file at a path, folding case: a later profile's different file there
@@ -2142,6 +2147,8 @@ def verify_emulator(
         bluemsx's Machines/ tree holds msx-emu's coleco.rom path in a
         --system colecovision pack."""
         for dd in deliverable(profile.get("data_directories", []), standalone):
+            if required_only and dd.get("required") is False:
+                continue
             cache = (data_registry or {}).get(dd.get("ref", ""), {}).get("local_cache", "")
             if not cache or not os.path.isdir(cache):
                 continue
@@ -2170,6 +2177,17 @@ def verify_emulator(
         )
         if system_ids:
             files = [fe for fe in files if entry_serves_systems(fe, profile, system_ids)]
+        if required_only:
+            optional_archives = {
+                fe["archive"] for fe in files
+                if fe.get("archive")
+                and archive_is_optional(archive_entry(fe["archive"], files), fe["archive"], files)
+            }
+            files = [
+                fe for fe in files
+                if fe.get("archive") not in optional_archives
+                and (fe.get("archive") or fe.get("required") is not False)
+            ]
         if region_drops:
             # Loose files only, as the emulator pack: an archive ships whole,
             # and dropping kronos' regional stvbios.zip members rebuilt the
@@ -2185,6 +2203,8 @@ def verify_emulator(
         for dd in deliverable(
             profile.get("data_directories", []), standalone
         ):
+            if required_only and dd.get("required") is False:
+                continue
             ref = dd.get("ref", "")
             if not ref:
                 continue
@@ -2516,6 +2536,7 @@ def verify_system(
     standalone: bool = False,
     regions: list[str] | None = None,
     platforms_dir: str = _REPO_PLATFORMS,
+    required_only: bool = False,
 ) -> dict:
     """Verify files for all emulators supporting given system IDs."""
     profiles = load_emulator_profiles(emulators_dir)
@@ -2541,7 +2562,7 @@ def verify_system(
 
     return verify_emulator(
         matching, emulators_dir, db, standalone, regions=regions,
-        platforms_dir=platforms_dir, system_ids=system_ids,
+        platforms_dir=platforms_dir, system_ids=system_ids, required_only=required_only,
     )
 
 
@@ -2680,6 +2701,7 @@ def _refuse_listing_narrowings(
         ("--destinations", args.destinations),
         ("--offline", args.offline),
         ("--no-game-data", getattr(args, "no_game_data", False)),
+        ("--required-only", args.required_only),
     ):
         if on:
             parser.error(f"{flag} is incompatible with {listing}")
@@ -2708,6 +2730,8 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
          "--no-game-data requires --platform or --all"),
         (args.destinations and getattr(args, "region", None),
          "--destinations describes the full pack and is incompatible with --region"),
+        (args.destinations and args.required_only,
+         "--destinations describes the full pack and is incompatible with --required-only"),
     ):
         if refused:
             parser.error(message)
@@ -2776,6 +2800,11 @@ def _main(held: contextlib.ExitStack) -> None:
         help="Report on the pack built with --no-game-data: game data left "
         "out and named, firmware only",
     )
+    parser.add_argument(
+        "--required-only", action="store_true",
+        help="Report on the pack built with --required-only: files the code "
+        "runs without are left out",
+    )
     parser.add_argument(
         "--db", default=DEFAULT_DB,
         help="database file to read (default: %(default)s)",
@@ -2867,6 +2896,7 @@ def _main(held: contextlib.ExitStack) -> None:
         result = verify_emulator(
             names, args.emulators_dir, db, args.standalone,
             regions=requested_regions, platforms_dir=args.platforms_dir,
+            required_only=args.required_only,
         )
         if args.json:
             result["details"] = _json_details(result["details"], args.verbose)
@@ -2881,6 +2911,7 @@ def _main(held: contextlib.ExitStack) -> None:
         result = verify_system(
             system_ids, args.emulators_dir, db, args.standalone,
             regions=requested_regions, platforms_dir=args.platforms_dir,
+            required_only=args.required_only,
         )
         if args.json:
             result["details"] = _json_details(result["details"], args.verbose)
@@ -2950,6 +2981,7 @@ def _main(held: contextlib.ExitStack) -> None:
             regions=requested_regions,
             system_filter=platform_systems,
             no_game_data=getattr(args, "no_game_data", False),
+            required_only=args.required_only,
         )
         names = [
             load_platform_config(p, args.platforms_dir).get("platform", p)