chore: merge engine and tooling batch

scripts/artifacts.py

@@ -12,6 +12,7 @@ import contextlib
 import json
 import os
 import tempfile
+import threading
 import re
 
 
@@ -62,12 +63,26 @@ def write_if_changed(
     return True
 
 
+_UMASK_LOCK = threading.Lock()
+
+
+def process_umask() -> int:
+    """The process umask, read without letting a second thread see it at 0.
+
+    Reading the umask sets it, so it is put back at once. Two threads doing
+    that unguarded interleave as A sets 0 and reads 022, B sets 0 and reads 0,
+    A restores 022, B restores 0: the process keeps umask 0 and every file it
+    creates afterwards is world-writable.
+    """
+    with _UMASK_LOCK:
+        umask = os.umask(0)
+        os.umask(umask)
+    return umask
+
+
 def plain_file_mode() -> int:
     """The mode open() gives a new file: 0o666 less the process umask."""
-    # Reading the umask sets it, so it is put back at once.
-    umask = os.umask(0)
-    os.umask(umask)
-    return 0o666 & ~umask
+    return 0o666 & ~process_umask()
 
 
 def set_plain_mode(path: str | os.PathLike) -> None:

scripts/auto_fetch.py

@@ -107,7 +107,10 @@ def find_missing(config: dict, db: dict) -> list[dict]:
                 "destination": file_entry.get("destination", name),
             })
     for u in result["undeclared_files"]:
-        if u.get("in_repo") or not u.get("required"):
+        # A file installed with its emulator is not an acquisition target:
+        # verify leaves it out of the missing count, and bundled_only.py
+        # --strict refuses a copy that only bundled entries reach.
+        if u.get("in_repo") or not u.get("required") or u.get("bundled"):
             continue
         missing.append({
             "name": u["name"],

scripts/check_freshness.py

@@ -524,7 +524,7 @@ def check_native(
             )
             if reason:
                 reason += (
-                    f", export_native.py --platform {name} --refresh-cache"
+                    f", python scripts/export_native.py --platform {name} --refresh-cache"
                     " fetches it, then the export is written again"
                 )
             else:
@@ -532,7 +532,7 @@ def check_native(
                     name, exporter_class, truth_dir, platforms_dir, cache_dir, published
                 )
                 if reason:
-                    reason += f", export_native.py --platform {name} writes it again"
+                    reason += f", python scripts/export_native.py --platform {name} writes it again"
         findings.append(
             Finding(
                 "native",
@@ -974,7 +974,7 @@ def check_coreinfo(
     if recorded is None:
         findings.append(Finding(
             "coreinfo", "reviewed .info", UNKNOWN,
-            detail=f"no {record_path.name}: run check_freshness.py --record-coreinfo",
+            detail=f"no {record_path.name}: run python scripts/check_freshness.py --record-coreinfo",
         ))
         return findings
     findings.extend(
@@ -982,7 +982,7 @@ def check_coreinfo(
             "coreinfo", name, STALE,
             local=(recorded.get(name) or "not recorded")[:12], remote=blobs[name][:12],
             detail=(".info changed since reviewed: read its firmware, then "
-                    f"check_freshness.py --record-coreinfo {name}"),
+                    f"python scripts/check_freshness.py --record-coreinfo {name}"),
         )
         for name in coreinfo_drift(blobs, recorded, profiles)
     )
@@ -1456,11 +1456,122 @@ def check_source_paths(profiles: dict) -> Finding:
         "source-paths.json",
         STALE if problems else OK,
         local=f"{len(recorded)} trees",
-        detail=(", ".join(problems) + ": run scripts/sitesources.py --fetch") if problems
+        detail=(", ".join(problems) + ": run python scripts/sitesources.py --fetch") if problems
         else "every pinned tree is recorded, and only those",
     )
 
 
+def build_variable(lines: list[str], variable: str) -> str | None:
+    """The value the first assignment to ``variable`` gives in a build file.
+
+    Read as a Buildroot .mk (`X_VERSION = sha`), a LibreELEC package.mk
+    (`PKG_VERSION="sha"`) or a RetroPie scriptmodule (`rp_module_repo="git
+    URL tag"`, `local version="20250829"`): quotes and a trailing comment go,
+    the rest is compared as written.
+    """
+    assignment = re.compile(
+        rf"^\s*(?:local\s+|export\s+)?{re.escape(variable)}\s*[:?+]?=\s*(.*)$"
+    )
+    for line in lines:
+        match = assignment.match(line)
+        if not match:
+            continue
+        value = match.group(1).strip()
+        if value[:1] in "\"'":
+            end = value.find(value[0], 1)
+            return value[1:end] if end > 0 else value[1:]
+        return value.split(" #", 1)[0].strip()
+    return None
+
+
+def pin_build_revision(reference: dict, platforms_dir: Path) -> str:
+    """The revision of the platform repository its YAML was last scraped at,
+    '' when the platform is not registered or reads another repository."""
+    path = platforms_dir / f"{reference.get('platform', '')}.yml"
+    repo = upstream.parse_repo(str(reference.get("repo") or ""))
+    if repo is None or not path.is_file():
+        return ""
+    with path.open(encoding="utf-8") as fh:
+        source = str((yaml_load(fh) or {}).get("source") or "")
+    if f"/{repo.slug.lower()}/" not in source.lower() + "/":
+        return ""
+    return export_native.source_revision(source)
+
+
+def check_pin_builds(
+    profiles: dict, platforms_dir: Path, cache_dir: str, offline: bool,
+) -> list[Finding]:
+    """Each `pin_build:` read again where the platform now stands.
+
+    A pin_frozen pin is held because one platform builds that revision, and
+    nothing read the build file again: Recalbox moving mame0278 to another
+    commit left the profile describing a build no platform ships. The file
+    is read at the revision the platform's own YAML was scraped at, or at
+    the repository's branch head for a platform this repository does not
+    scrape, and its variable compared with the value the profile recorded.
+    """
+    findings: list[Finding] = []
+    unreferenced = sorted(
+        name for name, profile in profiles.items()
+        if profile.get("pin_frozen") and not profile.get("pin_build")
+    )
+    if unreferenced:
+        findings.append(Finding(
+            "profiles", "pin_frozen without pin_build", SKIPPED,
+            local=f"{len(unreferenced)} profiles",
+            detail="no platform build file names their revision: "
+                   + ", ".join(unreferenced[:SHOWN_NAMES])
+                   + (", ..." if len(unreferenced) > SHOWN_NAMES else ""),
+            items=unreferenced,
+        ))
+    for name, profile in sorted(profiles.items()):
+        for reference in profile.get("pin_build") or []:
+            subject = f"{name} ({reference['platform']})"
+            recorded = str(reference["value"])
+            revision = pin_build_revision(reference, platforms_dir)
+            if revision == str(reference["revision"]):
+                findings.append(Finding(
+                    "profiles", subject, OK, local=recorded[:12],
+                    detail=f"read where platforms/{reference['platform']}.yml is",
+                ))
+                continue
+            repo = upstream.parse_repo(str(reference["repo"]))
+            try:
+                at = revision or upstream.resolve_head(repo, cache_dir, offline)
+                lines = upstream.fetch_file(repo, at, reference["path"], cache_dir, offline) \
+                    if at else None
+            except upstream.NotCachedError:
+                findings.append(Finding("profiles", subject, SKIPPED, local=recorded[:12],
+                                        detail="offline, the build file is not cached"))
+                continue
+            except upstream.UpstreamError as exc:
+                findings.append(Finding("profiles", subject, UNKNOWN, local=recorded[:12],
+                                        detail=str(exc)))
+                continue
+            if not at:
+                findings.append(Finding("profiles", subject, UNKNOWN, local=recorded[:12],
+                                        detail=f"{repo.slug} answered no branch head"))
+                continue
+            current = None if lines is None else build_variable(lines, reference["variable"])
+            if current == recorded:
+                findings.append(Finding(
+                    "profiles", subject, OK, local=recorded[:12], remote=at[:12],
+                    detail=f"{reference['path']} still builds it",
+                ))
+                continue
+            findings.append(Finding(
+                "profiles", subject, STALE, local=recorded[:12],
+                remote=(current or "gone")[:12],
+                detail=(
+                    f"{reference['path']} is gone at {at[:12]}" if lines is None
+                    else f"{reference['variable']} is no longer set there" if current is None
+                    else f"{reference['platform']} now builds {current} "
+                         f"({reference['path']} at {at[:12]}): the pin follows the build"
+                ),
+            ))
+    return findings
+
+
 def check_profiles(run: bool, offline: bool) -> list[Finding]:
     if not run:
         return [
@@ -1468,7 +1579,7 @@ def check_profiles(run: bool, offline: bool) -> list[Finding]:
                 "profiles",
                 "profile_sync",
                 SKIPPED,
-                detail="pass --profiles, or run profile_sync.py --all --triage",
+                detail="pass --profiles, or run python scripts/profile_sync.py --all --triage",
             )
         ]
     cmd = [
@@ -1708,6 +1819,7 @@ def main() -> int:
         findings.extend(check_ci(args.cache_dir, args.offline))
     if "profiles" in areas:
         findings.append(check_source_paths(profiles))
+        findings.extend(check_pin_builds(profiles, platforms_dir, args.cache_dir, args.offline))
         findings.extend(check_profiles(args.profiles, args.offline))
 
     if args.json:

scripts/commit_rules.py

@@ -21,6 +21,14 @@ REFUSED_WORDS = re.compile(
     r"comprehensive|implement)",
     re.IGNORECASE,
 )
+# A numbered pointer at a private review ("audit 24", "review 3", "round 5")
+# names notes no clone holds. tests/test_prose_style.py refuses the same tags
+# in tracked files, and history cannot be rewritten once pushed.
+REVIEW_TAG = re.compile(
+    r"\b(audit|review|round|batch|finding)s? #?\d+|\baudit (batch|findings?)\b"
+    r"|[a-z]+_audit_\d+",
+    re.IGNORECASE,
+)
 CO_AUTHOR = re.compile(r"^co-authored-by", re.IGNORECASE | re.MULTILINE)
 
 
@@ -35,6 +43,8 @@ def problems(message: str) -> list[str]:
     text = "\n".join(line for line in message.splitlines() if not line.startswith("#"))
     if REFUSED_WORDS.search(text):
         found.append("the message uses a word this repository's history does not carry")
+    if REVIEW_TAG.search(text):
+        found.append("say what the change does, not which review asked for it")
     if CO_AUTHOR.search(message):
         found.append("no Co-Authored-By trailer here")
     return found

scripts/common.py

@@ -691,7 +691,7 @@ def load_target_config(
             # --target but have no --list-targets of their own.
             + suggest_names(
                 [target], sorted(alias_index),
-                f"verify.py --platform {platform_name} --list-targets", len(targets),
+                f"python scripts/verify.py --platform {platform_name} --list-targets", len(targets),
             )
         )
 
@@ -1870,12 +1870,12 @@ def select_emulator_profiles(
     all_profiles: dict,
     standalone: bool = False,
 ) -> list[tuple[str, dict]]:
-    """Resolve named profiles, refusing the ones that answer for others.
+    """Resolve named profiles, refusing the ones that answer for no core.
 
-    An alias is the same binary under another name and a launcher only
-    starts an emulator, so neither has requirements of its own: naming one
-    is a question about the wrong profile, and the error says which one to
-    ask instead. Raising rather than exiting lets the verifier stop and the
+    An alias is the same binary under another name, so naming one selects
+    the profile it is an alias of, said on stderr. A launcher only starts an
+    emulator and has no requirements of its own: naming one is a question
+    about the wrong profile, and the error says so. Raising rather than exiting lets the verifier stop and the
     builder return empty-handed from one rule.
     """
     selected: list[tuple[str, dict]] = []
@@ -1916,11 +1916,17 @@ def select_emulator_profiles(
                 ),
                 None,
             )
-            raise ProfileSelectionError(
-                f"{name} is an alias of {target}: use --emulator {target}"
-                if target
-                else f"{name} is an alias with no profile of its own to name"
-            )
+            if not target or target not in all_profiles:
+                raise ProfileSelectionError(
+                    f"{name} is an alias with no profile of its own to name"
+                )
+            # The name a user reads in RetroArch or on the buildbot: the tool
+            # knows the profile, so it says which and carries on.
+            print(f"{name} is {target}: using its profile", file=sys.stderr)
+            name, profile = target, all_profiles[target]
+            kind = profile.get("type", "libretro")
+            if any(key == name for key, _ in selected):
+                continue
         if kind == "launcher":
             raise ProfileSelectionError(
                 f"{name} is a launcher: use the emulator it launches"
@@ -3250,26 +3256,55 @@ def build_target_cores_cache(
     """
     cache: dict[str, set[str] | None] = {}
     skip: list[str] = []
+    notes: list[str] = []
     for p in platforms:
         try:
             cache[p] = load_target_config(p, target, platforms_dir)
         except FileNotFoundError:
             if not is_all:
                 raise
-            print(f"INFO: Skipping {p}: no hardware targets")
+            notes.append(f"INFO: Skipping {p}: no hardware targets")
             skip.append(p)
         except ValueError as e:
             if is_all:
-                print(f"INFO: Skipping {p}: {e}")
+                notes.append(f"INFO: Skipping {p}: {e}")
                 skip.append(p)
             else:
                 raise
     kept = [p for p in platforms if p not in skip]
     if is_all and platforms and not kept:
-        raise ValueError(f"unknown target '{target}': no platform has it")
+        # The seventeen per-platform lines buried the one suggestion that
+        # mattered: the refusal names the closest target across them all.
+        raise ValueError(_unknown_target_message(target, platforms, platforms_dir))
+    for note in notes:
+        print(note)
     return cache, kept
 
 
+def _unknown_target_message(target: str, platforms: list[str], platforms_dir: str) -> str:
+    """A target no platform has, with the closest names any of them knows."""
+    import difflib  # noqa: PLC0415
+
+    targets_dir = os.path.join(platforms_dir, "targets")
+    owners: dict[str, list[str]] = {}
+    for platform in platforms:
+        target_file = os.path.join(targets_dir, f"{platform}.yml")
+        if not os.path.exists(target_file):
+            continue
+        with open(target_file) as f:
+            targets = (yaml_load(f) or {}).get("targets", {})
+        overrides = _target_overrides(platform, targets_dir)
+        for name in _target_alias_index(targets, overrides):
+            owners.setdefault(name, []).append(platform)
+    message = f"unknown target '{target}': no platform has it."
+    close = difflib.get_close_matches(target, sorted(owners), n=3, cutoff=0.6)
+    if close:
+        named = ", ".join(f"{n} ({', '.join(sorted(owners[n]))})" for n in close)
+        return f"{message} Did you mean: {named}?"
+    return (f"{message} python scripts/verify.py --platform <name> --list-targets "
+            "lists the targets of one platform.")
+
+
 # Re-exported so the existing call sites keep working while the
 # modules above become the place to import from.
 from safeparse import (  # noqa: E402,F401

scripts/exporter/candidates.py

@@ -375,6 +375,7 @@ class EntryCategories:
         # entry with a proof walks its reader's whole profile, and a MAME
         # profile's archive members were hashed again for each.
         self._accepted: dict[int, set[str]] = {}
+        self._entry_index: dict[tuple[str, str], tuple[dict, dict]] = {}
         # One index per core and per kind, so a proof names its reader.
         self._readers: list[tuple[str, str, ReadEvidence]] = []
         for key in sorted(examined):
@@ -418,6 +419,24 @@ class EntryCategories:
             accepted = self._accepted[id(profile_entry)] = _entry_hashes(profile_entry)
         return accepted
 
+    def _index_of(self, key: str, kind: str) -> tuple[dict, dict]:
+        """A profile's entries of one kind by casefolded name and by accepted
+        hash, each list in file order. Built once per profile and kind: a
+        scan per platform entry cost 2 s of verify --all on MAME alone."""
+        index = self._entry_index.get((key, kind))
+        if index is None:
+            by_name: dict[str, list[tuple[int, dict]]] = {}
+            by_hash: dict[str, list[tuple[int, dict]]] = {}
+            for position, f in enumerate(self._profiles[key].get("files") or []):
+                if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA):
+                    continue
+                for n in {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n}:
+                    by_name.setdefault(n, []).append((position, f))
+                for value in self._accepted_by(f):
+                    by_hash.setdefault(value, []).append((position, f))
+            index = self._entry_index[(key, kind)] = (by_name, by_hash)
+        return index
+
     def _matching(self, key: str, kind: str, entry: dict) -> list[dict]:
         """The profile entries of `key` that read the platform entry.
 
@@ -434,19 +453,28 @@ class EntryCategories:
             if n
         }
         declared = _declared_hashes(entry)
-        by_hash: list[dict] = []
-        by_name: list[dict] = []
-        for f in self._profiles[key].get("files") or []:
-            if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA):
-                continue
-            cited = {k: f[k] for k in ("name", "source_ref") if f.get(k)}
-            if declared and _accepts(self._accepted_by(f), declared):
-                by_hash.append(cited)
-                continue
-            own = {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n}
-            if own & names and not (declared and _contradicts(f, declared)):
-                by_name.append({**cited, "by": "name"})
-        return by_hash or by_name
+        by_name_index, by_hash_index = self._index_of(key, kind)
+
+        def cited(f: dict) -> dict:
+            return {k: f[k] for k in ("name", "source_ref") if f.get(k)}
+
+        found: dict[int, dict] = {}
+        for values in declared.values():
+            for value in values:
+                for position, f in by_hash_index.get(value, ()):
+                    found[position] = f
+                if _CUT_MD5 <= len(value) < _FULL_MD5:
+                    for accepted, hits in by_hash_index.items():
+                        if accepted.startswith(value):
+                            found.update(hits)
+        if found:
+            return [cited(found[p]) for p in sorted(found)]
+        named = {position: f for n in names for position, f in by_name_index.get(n, ())}
+        return [
+            {**cited(named[p]), "by": "name"}
+            for p in sorted(named)
+            if not (declared and _contradicts(named[p], declared))
+        ]
 
     def explain(self, entry: dict, destination: str, system: str) -> dict:
         """The category of an entry and what establishes it."""

scripts/exporter/recalbox_exporter.py

@@ -245,6 +245,13 @@ class Exporter(BaseExporter):
 
         return f"    <bios {' '.join(attrs)} />"
 
+    def written_list(self, fe: NativeFile, field_name: str) -> list[str] | None:
+        """md5 is the comma list Bios.cpp splits and compares one by one, so
+        a list the truth extended is a change the report names."""
+        if field_name == "md5" and len(fe.hashes("md5")) > 1:
+            return fe.hashes("md5")
+        return None
+
     def states(self, fe: NativeFile, field_name: str) -> bool:
         """An element it cannot rewrite is written back as Recalbox wrote it.
 

scripts/exporter/retrodeck_exporter.py

@@ -96,6 +96,13 @@ class Exporter(BaseExporter):
             return fe.hash(field_name)
         return super().written(fe, field_name)
 
+    def written_list(self, fe: NativeFile, field_name: str) -> list[str] | None:
+        """Several md5 values are a JSON array in the manifest, as _entry
+        writes them: a list the truth extended is a change the report names."""
+        if field_name == "md5" and len(fe.hashes("md5")) > 1:
+            return fe.hashes("md5")
+        return None
+
     def states(self, fe: NativeFile, field_name: str) -> bool:
         if id(fe) in getattr(self, "_appended", ()):
             return False

scripts/fileless_audit.py

@@ -128,7 +128,7 @@ def main() -> None:
     if unknown:
         raise SystemExit(
             "ERROR: unknown profile " + ", ".join(unknown) + ". "
-            + suggest_names(unknown, known, "verify.py --list-emulators")
+            + suggest_names(unknown, known, "python scripts/verify.py --list-emulators")
         )
     names = args.emulators
     if args.all:

scripts/generate_event_schema.py

@@ -13,6 +13,8 @@ import json
 import sys
 from pathlib import Path
 
+from artifacts import write_text_atomic
+
 ROOT = Path(__file__).resolve().parent.parent
 OUTPUT = ROOT / "schemas" / "install-events.schema.json"
 
@@ -105,7 +107,7 @@ def main(argv: list[str] | None = None) -> int:
         print(f"{args.output} differs from EVENT_KEYS: run "
               "python scripts/generate_event_schema.py", file=sys.stderr)
         return 1
-    args.output.write_text(text, encoding="utf-8")
+    write_text_atomic(str(args.output), text)
     print(f"Wrote {args.output}")
     return 0
 

scripts/generate_games_index.py

@@ -59,6 +59,7 @@ from romfolders import (  # noqa: E402
     folder_names,
     load_setups,
 )
+from artifacts import write_text_atomic  # noqa: E402
 from safeparse import yaml_load  # noqa: E402
 
 import install  # noqa: E402
@@ -609,16 +610,60 @@ def print_coverage(coverage: dict, files: dict[Path, str]) -> None:
     print(f"{len(files)} files, {raw:,} bytes, {packed:,} bytes gzip")
 
 
+def _sample(items: list[str], limit: int = 3) -> str:
+    shown = ", ".join(items[:limit])
+    return f"{shown}, +{len(items) - limit} more" if len(items) > limit else shown
+
+
+def describe_change(old: str | None, new: str) -> list[str]:
+    """What differs between a published document and the one render() makes.
+
+    The documents are single-line JSON of up to 550 KB: STALE alone left a
+    diff by hand to find that a manifest gained one file."""
+    if old is None:
+        return ["not written yet"]
+    try:
+        before, after = json.loads(old), json.loads(new)
+    except ValueError:
+        return ["the published file is not valid JSON"]
+    if not (isinstance(before, dict) and isinstance(after, dict)):
+        return ["document changed"]
+    parts = []
+    for key in sorted(set(before) | set(after)):
+        a, b = before.get(key), after.get(key)
+        if a == b:
+            continue
+        if isinstance(a, list) and isinstance(b, list):
+            olds = {json.dumps(x, sort_keys=True) for x in a}
+            news = {json.dumps(x, sort_keys=True) for x in b}
+            added = sorted(news - olds)
+            removed = sorted(olds - news)
+            change = []
+            if added:
+                change.append(f"{len(added)} added ({_sample(added)})")
+            if removed:
+                change.append(f"{len(removed)} removed ({_sample(removed)})")
+            parts.append(f"{key}: {', '.join(change) or 'reordered'}")
+        elif isinstance(a, dict) and isinstance(b, dict):
+            changed = sorted(k for k in set(a) | set(b) if a.get(k) != b.get(k))
+            parts.append(f"{key}: {len(changed)} changed ({_sample(changed)})")
+        else:
+            parts.append(f"{key} changed" if key in before and key in after
+                         else f"{key} {'added' if key in after else 'removed'}")
+    return parts or ["formatting changed"]
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
     parser.add_argument("--output-dir", type=Path, default=OUTPUT,
                         help=f"where the index is written (default {OUTPUT})")
-    parser.add_argument("--check", action="store_true", help="exit 1 when a file would change")
+    parser.add_argument("--check", action="store_true",
+                        help="exit 1 when a file would change, naming what differs (implies --quiet)")
     parser.add_argument("--quiet", action="store_true", help="print only failures")
     args = parser.parse_args()
 
     files, coverage = render(args.output_dir)
-    if not args.quiet:
+    if not (args.quiet or args.check):
         print_coverage(coverage, files)
     failures = floor_failures(coverage)
     for failure in failures:
@@ -628,14 +673,22 @@ def main() -> int:
     ) if args.output_dir.is_dir() else []
     changed = [p for p, t in files.items() if not p.exists() or p.read_text(encoding="utf-8") != t]
     if args.check:
-        for path in changed + stale:
+        for path in changed:
             print(f"STALE {path}", file=sys.stderr)
+            current = path.read_text(encoding="utf-8") if path.exists() else None
+            for line in describe_change(current, files[path]):
+                print(f"  {line}", file=sys.stderr)
+        for path in stale:
+            print(f"STALE {path}", file=sys.stderr)
+            print("  no longer generated", file=sys.stderr)
+        if changed or stale:
+            print("Run python scripts/generate_games_index.py to write them.", file=sys.stderr)
         return 1 if changed or stale or failures else 0
     for path in stale:
         path.unlink()
     for path in changed:
         path.parent.mkdir(parents=True, exist_ok=True)
-        path.write_text(files[path], encoding="utf-8")
+        write_text_atomic(str(path), files[path])
     print(f"{len(changed)} written, {len(stale)} removed")
     return 1 if failures else 0
 

scripts/generate_installer_doc.py

@@ -45,6 +45,8 @@ import zipfile
 from pathlib import Path, PurePosixPath
 from typing import Callable
 
+from artifacts import write_text_atomic
+
 ROOT = Path(__file__).resolve().parent.parent
 PAGE = ROOT / "wiki" / "installer.md"
 INSTALLER = ROOT / "install.py"
@@ -647,7 +649,7 @@ def main(argv: list[str] | None = None) -> int:
         print(f"{args.page} differs from install.py and install/*.json: run "
               "python scripts/generate_installer_doc.py", file=sys.stderr)
         return 1
-    args.page.write_text(text, encoding="utf-8")
+    write_text_atomic(str(args.page), text)
     print(f"Wrote {args.page}")
     return 0
 

scripts/generate_launcher_index.py

@@ -22,6 +22,7 @@ import json
 import sys
 from pathlib import Path
 
+from artifacts import write_text_atomic
 from build_port import (
     BATOCERA_ZIP,
     MUOS_APP,
@@ -121,7 +122,7 @@ def main(argv: list[str] | None = None) -> int:
               "python scripts/generate_launcher_index.py", file=sys.stderr)
         return 1
     args.output.parent.mkdir(parents=True, exist_ok=True)
-    args.output.write_text(text, encoding="utf-8")
+    write_text_atomic(str(args.output), text)
     print(f"Wrote {args.output}")
     return 0
 

scripts/generate_pack.py

@@ -98,6 +98,7 @@ from validation import (
     validated_choice,
     agnostic_substitute,
     _build_validation_index,
+    _parse_validation,
     deliverable,
     with_archive_members,
     extra_alternatives,
@@ -623,6 +624,10 @@ def _matches_an_alternative(fe: dict, local_path: str, profiles: dict) -> bool:
     siblings = extra_alternatives(fe, profiles)
     if len(siblings) < 2:
         return False
+    # A sibling the code takes without a check (Raze's "BLOOD: Unknown
+    # Version") accepts whatever copy fills the slot.
+    if any(not _parse_validation(entry.get("validation")) for entry in siblings):
+        return True
     actual = compute_hashes(local_path, frozenset({"sha1", "md5", "crc32"}))
     for entry in siblings:
         for algorithm in ("sha1", "md5", "crc32"):
@@ -2725,7 +2730,8 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
             if args.platform:
                 print(
                     f"  {platform_name}: FAIL (no pack in {args.output_dir}; build it with "
-                    f"generate_pack.py --platform {platform_name} --output-dir {args.output_dir})"
+                    f"python scripts/generate_pack.py --platform {platform_name} "
+                    f"--output-dir {args.output_dir})"
                 )
                 all_ok = False
                 failing.append(f"{platform_name} (no pack)")
@@ -5077,14 +5083,22 @@ def _pack_archives(output_dir: str) -> list[str]:
     A part of a split pack holds a share of a platform by design and was
     checked against the whole when it was cut: judged as a pack it would
     fail conformance, and injecting a manifest would change published bytes.
+    Every name the builder gives holds `_BIOS_Pack`, then the system tag of
+    a --system platform pack (`_BIOS_Pack_Playstation.zip`): build_port.py
+    writes its launcher archives (RetroBIOS.zip) to the same dist/, and
+    taking every ZIP rewrote them with a manifest after build_port had
+    printed their hashes.
     """
     return sorted(
         name
         for name in os.listdir(output_dir)
-        if name.endswith(".zip") and not split_pack.is_part(name)
+        if _PACK_NAME.search(name) and not split_pack.is_part(name)
     )
 
 
+_PACK_NAME = re.compile(r"_BIOS_Pack(?:_[^/]+)?\.zip$")
+
+
 def verify_and_finalize_packs(
     output_dir: str,
     db: dict,

scripts/pipeline.py

@@ -22,7 +22,7 @@ Steps, numbered as the run prints them:
   4c   --manifest-targets           per-target core lists
   4d   bundled_only.py --strict     copies only an emulator install provides
   4e   generate_games_index.py      install/games/: what the games-on-card mode reads
-  5    consistency check            verify == pack: counts, paths, every emulator pack
+  5    consistency check            verify == pack: counts, paths, emulator and system packs
   6    --verify-packs               pack integrity, extraction and hashes
   7    generate_readme.py           README.md and CONTRIBUTING.md
        generate_installer_doc.py    run samples and tables of wiki/installer.md
@@ -518,7 +518,7 @@ def check_emulator_packs(
     emulators_dir: str = "emulators", platforms_dir: str = "platforms",
     db_path: str = "database.json",
 ) -> bool:
-    """Build every emulator pack in memory and compare it with verify --emulator.
+    """Build every emulator and system pack in memory and compare it with verify.
 
     Each profile in each build it has (libretro, standalone): the paths the
     pack fills with the profile's files against the ones verify credits. No
@@ -528,9 +528,11 @@ def check_emulator_packs(
     import verify  # noqa: PLC0415
     from common import (  # noqa: PLC0415
         ProfileSelectionError,
+        _norm_system_id,
         build_zip_contents_index,
         load_database,
         load_emulator_profiles,
+        profiles_for_systems,
         select_emulator_profiles,
     )
 
@@ -541,6 +543,8 @@ def check_emulator_packs(
     all_ok = True
     compared = 0
     for name in sorted(profiles):
+        if profiles[name].get("type") == "alias":
+            continue  # selects the profile it names, compared under that name
         for standalone in (False, True):
             try:
                 select_emulator_profiles([name], profiles, standalone)
@@ -564,7 +568,45 @@ def check_emulator_packs(
                 print("\n".join(lines))
                 all_ok = False
     print(f"  {compared} emulator packs compared, {'all equal' if all_ok else 'differences above'}")
-    return all_ok
+    # A system pack builds every profile of the system together: a placement
+    # rule that only goes wrong when two profiles share a path (the bluemsx
+    # tree beside msx-emu's files) never meets in a one-profile pack.
+    systems = sorted({
+        _norm_system_id(str(sid))
+        for profile in profiles.values()
+        if profile.get("type") not in ("launcher", "alias", "test")
+        for sid in profile.get("systems") or []
+    })
+    systems_ok = True
+    compared = 0
+    for sid in systems:
+        for standalone in (False, True):
+            matching = profiles_for_systems(profiles, [sid], standalone)
+            if len(matching) < 2:
+                continue
+            record = generate_pack.PackRecord()
+            with contextlib.redirect_stdout(io.StringIO()), \
+                    contextlib.redirect_stderr(io.StringIO()):
+                generate_pack.generate_emulator_pack(
+                    matching, emulators_dir, db, "bios", "", standalone, zip_contents,
+                    offline=True, platforms_dir=platforms_dir, record=record,
+                    system_ids=[sid],
+                )
+                report = verify.verify_emulator(
+                    matching, emulators_dir, db, standalone, platforms_dir=platforms_dir,
+                    system_ids=[sid],
+                )
+            compared += 1
+            label = f"--system {sid}{' --standalone' if standalone else ''}"
+            lines = _set_difference_lines(
+                label, set(report["pack_destinations"]), set(record.files)
+            )
+            if lines:
+                print("\n".join(lines))
+                systems_ok = False
+    print(f"  {compared} system packs compared, "
+          f"{'all equal' if systems_ok else 'differences above'}")
+    return all_ok and systems_ok
 
 
 class _Skipped:

scripts/profile_sync.py

@@ -1257,6 +1257,14 @@ def _follow_fork_base(
         base = upstream.fork_base(
             view.repo, view.head, source.repo, source.head, cache_dir, offline
         )
+        if base and base != view.pin:
+            # A pin newer than the merge base names code the port does not
+            # carry: the upstream is a build of its own, the Qt melonDS or the
+            # EasyRPG Player a standalone + libretro profile documents, and
+            # its HEAD stays the reference. melonds read 825 commits back.
+            moved = upstream.ancestry(view.repo, base, view.pin, cache_dir, offline)
+            if moved and moved[0] > 0 and moved[1] == 0:
+                base = None
         followed.append(
             replace(view, head=base, origin=f"{view.origin}, fork base of {source.repo.slug}")
             if base else view
@@ -2627,6 +2635,7 @@ def tree_diff(result: CompareResult, ref_dirs: set[str]) -> list[str]:
 
 
 DRIFT_WEIGHTS = {"GONE": 40, "CHANGED": 30, "AMBIGUOUS": 15, "SHIFTED": 1}
+PIN_WEIGHT = 50
 
 
 def drift_score(
@@ -2644,6 +2653,10 @@ def drift_score(
     if version is not None and not version.tag_matches_declared:
         if version.latest_release and version.latest_release != version.declared:
             score += 10
+    # A pin off its branch or a repository answering under another name is
+    # set again before any ref can be read, so each outranks a GONE ref:
+    # pcem ranked last at 0 with an orphaned upstream_commit.
+    score += PIN_WEIGHT * (len(report.orphaned) + len(report.moved_repos))
     score += min(commits, 20) // 4
     return score
 
@@ -4154,16 +4167,22 @@ def _refuse_dirty_tree(emulators_dir: str, names: list[str]) -> None:
         print(f"{e}. Pass --force to write anyway.", file=sys.stderr)
         raise SystemExit(1) from e
     if dirty:
+        # Another session may own the change, so the refusal names the files:
+        # --force writes over them.
+        one = len(dirty) == 1
         print(
-            f"{emulators_dir} carries uncommitted changes to the selected "
-            "profiles. Commit them first or pass --force.",
+            f"{', '.join(dirty)} {'has' if one else 'have'} uncommitted changes: "
+            f"commit {'it' if one else 'them'} or pass --force, which writes over "
+            f"{'it' if one else 'them'}.",
             file=sys.stderr,
         )
         raise SystemExit(1)
 
 
-def emulators_dir_is_dirty(emulators_dir: str, names: list[str] | None = None) -> bool:
-    """True when the profile directory carries uncommitted changes.
+def emulators_dir_is_dirty(emulators_dir: str, names: list[str] | None = None) -> list[str]:
+    """The files of the profile directory that carry uncommitted changes.
+
+    Paths read as the status prints them, from the repository root.
 
     ``names`` narrows the question to those profiles. Git is asked from
     inside the directory, so a profile tree that lives in another
@@ -4181,7 +4200,11 @@ def emulators_dir_is_dirty(emulators_dir: str, names: list[str] | None = None) -
         raise RuntimeError(
             f"cannot read the git status of {emulators_dir}: {result.stderr.strip()}"
         )
-    return bool(result.stdout.strip())
+    return sorted(
+        line[3:].split(" -> ")[-1].strip('"')
+        for line in result.stdout.splitlines()
+        if line.strip()
+    )
 
 
 def build_parser() -> argparse.ArgumentParser:
@@ -4414,7 +4437,7 @@ def select_profiles(profiles: dict, args: argparse.Namespace) -> dict:
             )
             raise SystemExit(
                 f"ERROR: unknown profile {', '.join(unknown)}. "
-                + suggest_names(unknown, listed, "verify.py --list-emulators")
+                + suggest_names(unknown, listed, "python scripts/verify.py --list-emulators")
             )
         return {name: profiles[name] for name in dict.fromkeys(names)}
 
@@ -4451,7 +4474,7 @@ def select_profiles(profiles: dict, args: argparse.Namespace) -> dict:
         if args.system:
             # Suggested from the spellings the named listing prints.
             systems = sorted(spelling for spelling, _ in system_index(profiles).values())
-            hint = " " + suggest_names([args.system], systems, "verify.py --list-systems")
+            hint = " " + suggest_names([args.system], systems, "python scripts/verify.py --list-systems")
         elif args.type_:
             kinds = sorted({str(p.get("type")) for p in profiles.values() if p.get("type")})
             hint = f" Types: {', '.join(kinds)}."
@@ -4604,8 +4627,14 @@ def _apply_writes(
             # The guard above read the prose under the pins the profile held.
             # A bump that moves upstream_commit can leave prose the report
             # judged in the fork under the new upstream pin (dolphin-mmjr2,
-            # hatari, xenios): read it again on what was written.
-            left = realign_prose(path, args.cache_dir, args.offline, dry_run=True)
+            # hatari, xenios): read it again on what was written. It reads
+            # the profile's own history, so it runs on the real path, and a
+            # read that fails (offline, rate limit) puts the profile back.
+            try:
+                left = realign_prose(path, args.cache_dir, args.offline, dry_run=True)
+            except BaseException:
+                path.write_bytes(original)
+                raise
             if moved and left:
                 path.write_bytes(original)
                 print(

scripts/provenance_report.py

@@ -66,7 +66,8 @@ def profile_needs(profiles: dict, db: dict) -> dict:
 
     Keyed by every hash a profile accepts, and, for an entry that declares
     a size and no hash, by its names casefolded. An entry with neither
-    cannot be told apart from any same-named dump and is left out.
+    cannot be told apart from any same-named dump and is left out, as is a
+    file the emulator installs itself (`bundled:`) on every platform.
     """
     files = db.get("files", {})
     indexes = db.get("indexes", {})
@@ -92,6 +93,10 @@ def profile_needs(profiles: dict, db: dict) -> dict:
             name = str(entry.get("name", ""))
             if not name or name.endswith("/"):
                 continue
+            # Installed with the emulator, so not an acquisition target,
+            # unless a platform package leaves it out.
+            if entry.get("bundled") and not entry.get("not_bundled_on"):
+                continue
             need = {"emulator": emulator, "name": name,
                     "required": bool(entry.get("required"))}
             hashes = {(field, v) for field in HASH_FIELDS for v in _values(entry, field)}

scripts/validation.py

@@ -8,6 +8,7 @@ truth data for reporting.
 from __future__ import annotations
 
 import os
+import re
 import zipfile
 from collections.abc import Callable, Iterator, Mapping, Sequence
 
@@ -161,6 +162,11 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
     Raises ValueError if two profiles declare conflicting values.
     """
     index: dict[str, dict] = {}
+    # A slot where one version is taken without any check (Raze's "BLOOD:
+    # Unknown Version", which holds the shareware RFF) accepts every copy for
+    # that core: its validated siblings are alternatives, not requirements.
+    unchecked: set[tuple[str, str, str]] = set()
+    checked: set[tuple[str, str, str]] = set()
     for emu_name, profile in profiles.items():
         if profile.get("type") in ("launcher", "alias"):
             continue
@@ -169,8 +175,11 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
             if not fname:
                 continue
             checks = _parse_validation(f.get("validation"))
+            slot = (fname, emu_name, f.get("path") or fname)
             if not checks:
+                unchecked.add(slot)
                 continue
+            checked.add(slot)
             if fname not in index:
                 index[fname] = {
                     "checks": set(),
@@ -276,6 +285,8 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
                     existing["source_ref"] = pe_entry["source_ref"]
             else:
                 pe[emu_name] = pe_entry
+    for fname, emu_name, _path in sorted(checked & unchecked):
+        index[fname]["rules"].append(_entry_rule(emu_name, {}, set()))
     # Convert sets to sorted tuples/lists for determinism
     for v in index.values():
         v["checks"] = sorted(v["checks"])
@@ -324,6 +335,23 @@ def _emulators_for_check(
     return sorted(result)
 
 
+_REASON = re.compile(r"^(\w+ mismatch): got (\S+), accepted \[(.*)\]$")
+
+
+def _merged_reason(reasons: list[str]) -> str:
+    """One reason for every entry of a core that turned the copy away.
+
+    Raze lists three BLOOD.RFF sizes and the message named the first one
+    alone. Reasons of one kind are merged into a single accepted list.
+    """
+    parsed = [_REASON.match(reason) for reason in reasons]
+    if all(parsed) and len({(m.group(1), m.group(2)) for m in parsed}) == 1:
+        accepted = sorted({v for m in parsed for v in m.group(3).split(",") if v},
+                          key=lambda v: (len(v), v))
+        return f"{parsed[0].group(1)}: got {parsed[0].group(2)}, accepted [{','.join(accepted)}]"
+    return reasons[0]
+
+
 def check_file_validation(
     local_path: str,
     filename: str,
@@ -363,7 +391,7 @@ def check_file_validation(
                 for rule in rules
             ]
             if all(reasons):
-                failures[emu] = reasons[0]
+                failures[emu] = _merged_reason(reasons)
         if not failures:
             return None
         first = sorted(failures)[0]

scripts/verify.py

@@ -1827,7 +1827,11 @@ def _omission_reasons(entries: list[dict]) -> str:
 def _print_undeclared_section(result: dict, verbose: bool) -> None:
     """Print cross-reference section for undeclared files used by cores."""
     undeclared = result.get("undeclared_files", [])
-    if not undeclared:
+    # The pack's core copies can come from no gap entry: recalbox
+    # --system nec-pc-engine packs six root syscard copies for geargrafx
+    # with an empty gap report, and returning here hid them.
+    packed = result.get("packed_extras")
+    if not undeclared and not (packed and any(packed.values())):
         return
 
     game_data, bios_files, unplaced = _split_undeclared(undeclared)
@@ -1850,7 +1854,6 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
     # The builder's count, one file per pack path, when the platform pass
     # replayed it. Counting entries said 1130 for RetroArch, 48 more than the
     # pack holds: rom.key alone was claimed by three Amiga cores.
-    packed = result.get("packed_extras")
     core_in_pack = (
         packed["bios"] if packed is not None else len(req_in_repo) + len(opt_in_repo)
     )
@@ -1880,7 +1883,7 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
         print(f"  Core files held but not packed: {len(unplaced)} "
               f"({_omission_reasons(unplaced)})")
 
-    if game_data:
+    if game_data or (packed and packed.get("game_data")):
         gd_missing = [u for u in game_data if not u["in_repo"]]
         gd_packed = (
             packed["game_data"] if packed is not None
@@ -2113,6 +2116,30 @@ def verify_emulator(
             return
         placed[dest.lower()] = (path, owner, dest)
         _register_path(dest, placed_files, placed_parents)
+
+    def occupy_trees(emu_name: str, profile: dict) -> None:
+        """The profile's data trees follow its files in the pack, which
+        writes them even when the system filter leaves it no file:
+        bluemsx's Machines/ tree holds msx-emu's coleco.rom path in a
+        --system colecovision pack."""
+        for dd in deliverable(profile.get("data_directories", []), standalone):
+            cache = (data_registry or {}).get(dd.get("ref", ""), {}).get("local_cache", "")
+            if not cache or not os.path.isdir(cache):
+                continue
+            dd_dest = dd.get("destination", "")
+            structure = profile.get("pack_structure")
+            prefix = (
+                structure.get("standalone" if standalone else "libretro", "")
+                if structure else ""
+            )
+            if prefix:
+                dd_dest = f"{prefix}/{dd_dest}" if dd_dest else prefix
+            for root, _dirs, names in os.walk(cache):
+                for fname in names:
+                    src = os.path.join(root, fname)
+                    rel = os.path.relpath(src, cache).replace(os.sep, "/")
+                    occupy(f"{dd_dest}/{rel}" if dd_dest else rel, src, emu_name)
+
     for emu_name, profile in sorted(selected):
         files = expand_agnostic_entries(
             expand_directory_entries(
@@ -2159,6 +2186,7 @@ def verify_emulator(
                     "ground_truth": [],
                 }
             )
+            occupy_trees(emu_name, profile)
             continue
 
         # Verify archives as units (e.g., neogeo.zip, aes.zip)
@@ -2421,24 +2449,7 @@ def verify_emulator(
                 ) > _SEVERITY_ORDER.get(prev_sev, 0):
                     file_severity[dest] = sev
 
-        # The profile's data trees follow its files in the pack.
-        for dd in deliverable(profile.get("data_directories", []), standalone):
-            cache = (data_registry or {}).get(dd.get("ref", ""), {}).get("local_cache", "")
-            if not cache or not os.path.isdir(cache):
-                continue
-            dd_dest = dd.get("destination", "")
-            structure = profile.get("pack_structure")
-            prefix = (
-                structure.get("standalone" if standalone else "libretro", "")
-                if structure else ""
-            )
-            if prefix:
-                dd_dest = f"{prefix}/{dd_dest}" if dd_dest else prefix
-            for root, _dirs, names in os.walk(cache):
-                for fname in names:
-                    src = os.path.join(root, fname)
-                    rel = os.path.relpath(src, cache).replace(os.sep, "/")
-                    occupy(f"{dd_dest}/{rel}" if dd_dest else rel, src, emu_name)
+        occupy_trees(emu_name, profile)
 
     counts = {
         Severity.OK: 0,

scripts/ziptools.py

@@ -14,7 +14,7 @@ import sys
 import tempfile
 import zipfile
 
-from artifacts import set_plain_mode
+from artifacts import process_umask, set_plain_mode
 from hashing import md5sum
 
 
@@ -251,9 +251,7 @@ def safe_extract_zip(
     """
     dest = os.path.realpath(dest_dir)
     os.makedirs(dest, exist_ok=True)
-    # Reading the umask sets it, so it is put back at once.
-    umask = os.umask(0)
-    os.umask(umask)
+    umask = process_umask()
     with zipfile.ZipFile(zip_path, "r") as zf:
         members = zf.infolist()
         if len(members) > max_members: