chore: merge tooling fixes, fourth pass

scripts/artifacts.py

@@ -19,6 +19,7 @@ import re
 _TIMESTAMP_PATTERNS = [
     re.compile(r'"generated_at":\s*"[^"]*"'),  # database.json
     re.compile(r'"imported_at":\s*"[^"]*"'),  # provenance snapshots
+    re.compile(r'"fetched":\s*"[^"]*"'),  # games/catalogs/
     re.compile(r"\*Auto-generated on [^*]*\*"),  # README.md
     re.compile(r"\*Generated on [^*]*\*"),  # docs site pages
     # The decorated pages carry the same stamp again as a rendered element,
@@ -208,6 +209,10 @@ def _strip_timestamps(text: str) -> str:
 class ArtifactLockBusy(RuntimeError):
     """Raised when another process already holds the artifact directory."""
 
+
+class ArtifactDirectoryMissing(FileNotFoundError):
+    """A reader named a directory that does not exist."""
+
 # A run that holds artifact directories for its whole duration names them
 # here, os.pathsep apart, for the steps it spawns: they would otherwise be
 # refused by it. A pipeline holds the repository root and its --output-dir.
@@ -227,6 +232,14 @@ def artifact_lock(directory: str, exclusive: bool = True) -> Iterator[None]:
     Writers take the lock exclusively, readers share it. On platforms
     without flock the lock is a no-op.
     """
+    # A reader creating the directory it was pointed at left a typo behind
+    # as a directory holding a .lock, and the next run with the same typo
+    # no longer said the path did not exist.
+    if not exclusive and not os.path.isdir(directory):
+        raise ArtifactDirectoryMissing(
+            f"{directory}: no such directory, pass the --output-dir the packs "
+            "were written to (default dist/)"
+        )
     try:
         import fcntl
     except ImportError:

scripts/bios_integrity.py

@@ -91,8 +91,19 @@ def sha1_of(path: Path) -> str:
     return digest.hexdigest()
 
 
+class GitFailed(RuntimeError):
+    """A git command failed: its empty output says nothing about the file."""
+
+
 def git(root: Path, *args: str) -> subprocess.CompletedProcess:
-    return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
+    """Run git in root. A failure raises: an unreadable index read as an empty
+    status made a session's working change look unmodified, and --apply wrote
+    HEAD's bytes over it."""
+    done = subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
+    if done.returncode:
+        message = done.stderr.decode(errors="replace").strip()
+        raise GitFailed(f"git {' '.join(args)} exited {done.returncode}: {message}")
+    return done
 
 
 def head_blob(root: Path, path: str) -> str:
@@ -197,7 +208,11 @@ def main() -> int:
     from common import load_database
 
     root = Path.cwd()
-    report = check(root, load_database(args.db), args.apply)
+    try:
+        report = check(root, load_database(args.db), args.apply)
+    except GitFailed as exc:
+        print(f"ERROR: {exc}. Nothing was restored past this point.", file=sys.stderr)
+        return 2
     if args.json:
         print(json.dumps(report.as_dict(), indent=2))
     else:

scripts/check_freshness.py

@@ -32,7 +32,6 @@ from __future__ import annotations
 
 import argparse
 import contextlib
-import hashlib
 import io
 import json
 import re
@@ -51,6 +50,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 
 import export_native  # noqa: E402
+import pin_installer  # noqa: E402
 import refresh_data_dirs  # noqa: E402
 import upstream  # noqa: E402
 from common import (  # noqa: E402
@@ -83,7 +83,6 @@ PYPI_URL = "https://pypi.org/pypi/{name}/json"
 NPM_URL = "https://registry.npmjs.org/{name}/latest"
 # The site's npm dependencies, pinned to exact versions and built by Deploy Site.
 SITE_PACKAGE = REPO_ROOT / "website" / "package.json"
-INSTALLERS = ("install.sh", "install.ps1")
 
 SHOWN_ITEMS = 4  # items named in a diff summary before ", ..."
 SHOWN_NAMES = 12  # core names named in an unresolved row before ", ..."
@@ -509,7 +508,10 @@ def check_native(
 ) -> list[Finding]:
     exporters = discover_exporters()
     index = cache_dir / export_native.SOURCES_INDEX
-    recorded = export_native.load_sources(cache_dir)
+    try:
+        recorded = export_native.load_sources(cache_dir)
+    except export_native.SourcesIndexUnreadable as exc:
+        return [Finding("native", index.name, ERROR, detail=str(exc))]
     findings: list[Finding] = []
     for name in list_registered_platforms(str(platforms_dir), include_archived=True):
         exporter_class = exporters.get(name)
@@ -1462,18 +1464,16 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]:
 
 def _check_installer_pins() -> Finding:
     """The bootstraps pin install.py by SHA-256, so a stale pin refuses every install."""
-    digest = hashlib.sha256((REPO_ROOT / "install.py").read_bytes()).hexdigest()
-    stale = []
-    for name in INSTALLERS:
-        text = (REPO_ROOT / name).read_text(encoding="utf-8", errors="replace")
-        if digest not in text:
-            stale.append(name)
+    stale = pin_installer.stale_pins(REPO_ROOT)
     return Finding(
         "ci",
         "install.py pin",
         STALE if stale else OK,
-        local=digest[:12],
-        detail=("pin outdated in " + ", ".join(stale)) if stale else "install.sh and install.ps1 match",
+        local=pin_installer.installer_sha256(REPO_ROOT)[:12],
+        detail=(
+            f"pin outdated in {', '.join(stale)}, {pin_installer.COMMAND} writes it"
+            if stale else "install.sh and install.ps1 match"
+        ),
     )
 
 
@@ -1757,15 +1757,18 @@ def render(findings: list[Finding]) -> str:
         # Columns are padded, never cut: the profile_sync row says in its
         # local and remote values what to re-read, and lost the end of it.
         local_width = min(max((len(f.local) for f in rows), default=0), 44)
+        remote_width = min(max((len(f.remote) for f in rows), default=0), 44)
+        # The arrow and the remote value, or as many blanks where a row has none.
+        remote_span = 4 + remote_width if remote_width else 0
         for f in rows:
             cols = f"  {f.status:7} {f.subject:{width}}"
             if f.remote:
-                cols += f"  {f.local:{local_width}} -> {f.remote}"
-            elif f.local or (f.detail and local_width):
-                # An arrow pointing at nothing read as a lost value. An empty
-                # local value still takes its column, or the detail starts
-                # where the other rows show theirs.
-                cols += f"  {f.local:{local_width}}"
+                cols += f"  {f.local:{local_width}} -> {f.remote:{remote_width}}"
+            elif f.local or (f.detail and (local_width or remote_span)):
+                # An arrow pointing at nothing read as a lost value. Empty
+                # local and remote values still take their columns, or the
+                # detail starts where the other rows show theirs.
+                cols += f"  {f.local:{local_width}}" + " " * remote_span
             if f.detail:
                 cols += f"  {f.detail}"
             lines.append(cols.rstrip())

scripts/common.py

@@ -3587,6 +3587,7 @@ from artifacts import (  # noqa: E402,F401
     copy_file_atomic,
     plain_file_mode,
     set_plain_mode,
+    ArtifactDirectoryMissing,
     ArtifactLockBusy,
     artifact_lock,
     hold_artifact_lock,
@@ -3634,7 +3635,7 @@ def run_cli(main: Callable[[], int | None]) -> None:
     try:
         code = main()
         sys.stdout.flush()
-    except (InputMissing, ArtifactLockBusy) as exc:
+    except (InputMissing, ArtifactLockBusy, ArtifactDirectoryMissing) as exc:
         print(f"ERROR: {exc}", file=sys.stderr)
         sys.exit(1)
     except BrokenPipeError:

scripts/export_native.py

@@ -36,6 +36,7 @@ from artifacts import file_lock, write_bytes_atomic, write_if_changed, write_tex
 import yaml
 
 from common import (
+    InputMissing,
     build_zip_contents_index,
     platform_arg,
     list_registered_platforms,
@@ -62,13 +63,27 @@ _USER_AGENT = "retrobios-exporter/1.0"
 _MAX_BYTES = 64 * 1024 * 1024
 
 
+class SourcesIndexUnreadable(InputMissing):
+    """The cache's URL index exists and does not read as JSON."""
+
+
 def _load_sources(index: Path | None) -> dict[str, str]:
+    """The URL each cached file was fetched from.
+
+    An absent index records nothing. An index that does not read raises:
+    read as empty it let every cached original pass for current, and the
+    next download rewrote it with that one entry.
+    """
     if index is None or not index.is_file():
         return {}
     try:
         return json.loads(index.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError):
-        return {}
+    except (OSError, json.JSONDecodeError) as exc:
+        raise SourcesIndexUnreadable(
+            f"{index} does not read ({exc}): remove it, then"
+            " python scripts/export_native.py --all --refresh-cache"
+            " downloads every original again and records its URL"
+        ) from exc
 
 
 def source_key(destination: Path, index: Path | None) -> str:

scripts/generate_readme.py

@@ -941,6 +941,10 @@ the most useful contribution.
    hash differs from the one already held under its name goes to that folder's
    `.variants/`.
 
+A ZIP that no list declares as a whole is read file by file, and the report
+says which of them the collection already holds: upload the files
+themselves, unzipped.
+
 From a clone, the file goes to `bios/Manufacturer/Console/filename`, and
 `python scripts/validate_pr.py path/to/file` names that folder for a file still
 outside `bios/`, and `--place` copies it there under its declared name. Given a

scripts/generate_truth.py

@@ -14,13 +14,13 @@ import sys
 
 sys.path.insert(0, os.path.dirname(__file__))
 from common import (
+    build_target_cores_cache,
     platform_arg,
     list_registered_platforms,
     load_data_dir_registry,
     load_database,
     load_emulator_profiles,
     load_platform_config,
-    load_target_config,
     require_directory,
     require_platform,
     require_yaml,
@@ -97,6 +97,19 @@ def main(argv: list[str] | None = None) -> None:
         require_platform(args.platform, args.platforms_dir)
         platforms = [args.platform]
 
+    # A named platform without that target is refused, as verify and
+    # generate_pack refuse it: skipped, it printed Done. and exited 0 with
+    # no model written. --all leaves such platforms out and says so.
+    target_cores_cache: dict[str, set[str] | None] = {}
+    if args.target:
+        try:
+            target_cores_cache, platforms = build_target_cores_cache(
+                platforms, args.target, args.platforms_dir, is_all=args.all
+            )
+        except (FileNotFoundError, ValueError) as exc:
+            print(f"ERROR: {exc}", file=sys.stderr)
+            sys.exit(1)
+
     # A target-narrowed model is a different artifact: written under the
     # platform's own name it overwrote the full one, and diff_truth then
     # compared a narrowed model against the whole scrape. Its directory is
@@ -109,17 +122,7 @@ def main(argv: list[str] | None = None) -> None:
                 output_dir, [name], args.target, args.platforms_dir
             )
         os.makedirs(output_dir, exist_ok=True)
-        target_cores: set[str] | None = None
-        if args.target:
-            try:
-                target_cores = load_target_config(
-                    name,
-                    args.target,
-                    args.platforms_dir,
-                )
-            except FileNotFoundError:
-                print(f"  {name}: no target config, skipped")
-                continue
+        target_cores = target_cores_cache.get(name)
 
         # Load platform config (with inheritance) and registry entry
         try:

scripts/pin_installer.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+"""Write the SHA-256 of install.py into the bootstraps that pin it.
+
+install.sh and install.ps1 refuse an installer whose hash is not the one
+they carry, so every change to install.py moves both pins in the same
+commit. --check writes nothing and exits 1 naming each stale pin.
+
+Usage:
+    python scripts/pin_installer.py
+    python scripts/pin_installer.py --check
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+INSTALLER = "install.py"
+# Each bootstrap and its one pin line, the value between the two groups.
+PINS: dict[str, re.Pattern[str]] = {
+    "install.sh": re.compile(r'^(DEFAULT_INSTALL_SHA256=")[0-9a-f]{64}(")', re.M),
+    "install.ps1": re.compile(r'^(\s*\$defaultInstallSha256 = ")[0-9a-f]{64}(")', re.M),
+}
+COMMAND = "python scripts/pin_installer.py"
+
+
+def installer_sha256(root: Path = ROOT) -> str:
+    return hashlib.sha256((root / INSTALLER).read_bytes()).hexdigest()
+
+
+def stale_pins(root: Path = ROOT) -> list[str]:
+    """The bootstraps whose pin is not the SHA-256 of install.py."""
+    digest = installer_sha256(root)
+    stale = []
+    for name, pin in PINS.items():
+        found = pin.search((root / name).read_bytes().decode("utf-8"))
+        if found is None:
+            raise ValueError(f"{name}: no installer pin line")
+        if found.group(0) != f"{found.group(1)}{digest}{found.group(2)}":
+            stale.append(name)
+    return stale
+
+
+def write_pins(root: Path = ROOT) -> list[str]:
+    """Rewrite each stale pin. Returns the bootstraps written."""
+    digest = installer_sha256(root)
+    written = []
+    for name in stale_pins(root):
+        path = root / name
+        text = path.read_bytes().decode("utf-8")
+        new, count = PINS[name].subn(rf"\g<1>{digest}\g<2>", text)
+        if count != 1:
+            raise ValueError(f"{name}: {count} installer pin lines, expected 1")
+        path.write_bytes(new.encode("utf-8"))
+        written.append(name)
+    return written
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
+    parser.add_argument("--check", action="store_true",
+                        help="write nothing, exit 1 if a pin is stale")
+    args = parser.parse_args()
+    if args.check:
+        stale = stale_pins()
+        for name in stale:
+            print(f"{name}: pin is not the SHA-256 of {INSTALLER}, {COMMAND} writes it",
+                  file=sys.stderr)
+        return 1 if stale else 0
+    for name in write_pins():
+        print(f"{name}: pinned {installer_sha256()}")
+    return 0
+
+
+if __name__ == "__main__":
+    from common import run_cli
+
+    run_cli(main)

scripts/refresh_data_dirs.py

@@ -71,6 +71,24 @@ def cache_lock(
     return _announced_lock(cache_dir.with_name(f".{cache_dir.name}.lock"), shared, cache_dir)
 
 
+def _wait_message(lock_path: Path, shared: bool, cache_dir: Path) -> str:
+    """Who to wait for, and, for a refresh, how not to.
+
+    The bare "another run holds it" named neither the lock nor its holder,
+    and the build waited with no limit.
+    """
+    message = (
+        f"waiting for {cache_dir}: another run holds {lock_path} "
+        f"(lsof {lock_path} names it)"
+    )
+    if not shared:
+        message += (
+            ". generate_pack.py --no-refresh-data, or --offline there and in"
+            " pipeline.py, builds from the cache as it is without waiting"
+        )
+    return message
+
+
 @contextlib.contextmanager
 def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[None]:
     """file_lock, saying so when it has to wait.
@@ -90,7 +108,7 @@ def _announced_lock(lock_path: Path, shared: bool, cache_dir: Path) -> Iterator[
         try:
             fcntl.flock(handle, mode | fcntl.LOCK_NB)
         except BlockingIOError:
-            log.warning("waiting for %s: another run holds it", cache_dir)
+            log.warning(_wait_message(lock_path, shared, cache_dir))
             fcntl.flock(handle, mode)
         try:
             yield

scripts/refresh_stale.py

@@ -34,7 +34,10 @@ has ended, and its export is rewritten even when no row named it.
 
 Surfaced only (no refresh): data rows that are not a _data_dirs.yml key
 (the registry itself, buildbot assets it does not track), coreinfo gaps, redump/no-intro/tosec packs,
-CI pins (install.py SHA-256, PyPI, pinned actions), profile_sync. Each needs
+CI pins (PyPI, pinned actions, while scripts/pin_installer.py writes the
+install.py SHA-256 pin), profile_sync, rom-folder
+pins (moved by hand, then python -m scripts.scraper.romfolders --setup NAME
+reads them again). Each needs
 a human read (new .info file, manual DAT download, PyPI version bump).
 
 Usage:
@@ -65,7 +68,9 @@ PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
 DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
 
 AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
-MANUAL_AREAS = ("coreinfo", "ci", "profiles")
+# A rom-folder pin is moved by hand in platforms/romfolders/_sources/,
+# then `python -m scripts.scraper.romfolders --setup NAME` reads it again.
+MANUAL_AREAS = ("romfolders", "coreinfo", "ci", "profiles")
 
 JOB_TIMEOUT = 1800  # 30 minutes per refresher, scrapers rarely exceed 10
 
@@ -397,6 +402,10 @@ def _command_for(
             "--force",
         ]
 
+    if area == "ci" and subject == "install.py pin":
+        # The pin is the hash of the committed install.py, nothing to judge.
+        return [sys.executable, "scripts/pin_installer.py"]
+
     if area == "catalogs":
         if subject == "mame recipes":
             return [
@@ -563,7 +572,8 @@ def main() -> int:
     parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
     parser.add_argument(
         "--only",
-        help="comma-separated areas to check: " + ",".join(AUTO_AREAS),
+        help="comma-separated areas to check: " + ",".join(AUTO_AREAS)
+        + " (refreshed), " + ",".join(MANUAL_AREAS) + " (listed for a human)",
     )
     parser.add_argument(
         "--dry-run",
@@ -579,7 +589,7 @@ def main() -> int:
     parser.add_argument(
         "--json",
         action="store_true",
-        help="machine-readable summary (jobs, surfaced, ignored)",
+        help="machine-readable summary (jobs, surfaced, ignored), the plan with --dry-run",
     )
     parser.add_argument(
         "--findings",
@@ -618,6 +628,22 @@ def main() -> int:
     registry = _load_platform_registry()
     jobs, surfaced, ignored = plan_jobs(findings, registry)
 
+    if args.dry_run and args.json:
+        print(json.dumps({
+            "planned": [
+                {
+                    "area": job.area,
+                    "subject": job.subject,
+                    "command": job.command,
+                    "after": sorted(job.after),
+                    "then": [list(command) for command in job.then],
+                }
+                for job in sorted(jobs, key=lambda j: (j.area, j.subject))
+            ],
+            "surfaced": surfaced,
+            "ignored_count": len(ignored),
+        }, indent=2))
+        return 0
     if args.dry_run:
         print(f"\n[plan] {len(jobs)} refreshable, {len(surfaced)} manual, "
               f"{len(ignored)} already clean/skipped")

scripts/scraper/arcade_parent_importer.py

@@ -49,7 +49,7 @@ from collections.abc import Callable, Iterator
 from dataclasses import dataclass, field
 from pathlib import Path
 
-from ..artifacts import write_text_atomic
+from ..artifacts import write_if_changed
 from ..common import load_emulator_profiles, run_cli
 from .romset_dat_importer import (
     LISTXML_CHUNK,
@@ -1114,14 +1114,6 @@ def dump(doc: dict) -> str:
     return json.dumps(doc, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
 
 
-def write_if_changed(path: Path, text: str) -> bool:
-    if path.is_file() and path.read_text(encoding="utf-8") == text:
-        return False
-    path.parent.mkdir(parents=True, exist_ok=True)
-    write_text_atomic(str(path), text)
-    return True
-
-
 def main() -> int:
     parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
     which = parser.add_mutually_exclusive_group(required=True)
@@ -1197,7 +1189,8 @@ def main() -> int:
             continue
         if not args.dry_run:
             target = args.output_dir / f"{profile}.json"
-            if write_if_changed(target, dump(doc)):
+            target.parent.mkdir(parents=True, exist_ok=True)
+            if write_if_changed(str(target), dump(doc)):
                 print(f"  wrote {target}")
     return 1 if failed else 0
 

scripts/scraper/emudeck_launchers.py

@@ -46,14 +46,18 @@ STORAGE = "${storagePath}"
 FLATPAK_DATA = "${HOME}/.var/app/"
 
 # EmuScripts file names that differ from the emulator profile they set up.
+# The platform and target scrapers both read them here.
 SCRIPT_EMULATORS = {
     "pcsx2qt": "pcsx2",
     "rpcs3legacy": "rpcs3",
     "cemuproton": "cemu",
     "rmg": "mupen64plus_next",
 }
-# The main RetroArch config script installs no emulator.
-NOT_EMULATORS = {"retroarch_maincfg"}
+# The main RetroArch config script installs no emulator, and emudeck-we's
+# emuDeckTemplate.ps1 is the skeleton new installers are copied from.
+NOT_EMULATORS = {"retroarch_maincfg", "template"}
+# EmuDeck's SteamOS installers are shell scripts, emudeck-we's PowerShell.
+SCRIPT_SUFFIXES = (".sh", ".ps1")
 
 # What the emulator does with a directory written into each key.
 DIRECTORY_KINDS = {
@@ -66,9 +70,10 @@ DIRECTORY_KINDS = {
 
 def emulator_of_script(file_name: str) -> str | None:
     """The emulator an EmuScripts file sets up, None for a helper script."""
-    if not file_name.endswith(".sh"):
+    stem, dot, suffix = file_name.rpartition(".")
+    if not dot or f".{suffix}" not in SCRIPT_SUFFIXES:
         return None
-    key = re.sub(r"^emuDeck", "", file_name[: -len(".sh")], flags=re.IGNORECASE).lower()
+    key = re.sub(r"^emuDeck", "", stem, flags=re.IGNORECASE).lower()
     if not key or key in NOT_EMULATORS:
         return None
     return SCRIPT_EMULATORS.get(key, key)

scripts/scraper/emudeck_scraper.py

@@ -469,9 +469,10 @@ class Scraper(BaseScraper):
 
         version = fetch_github_latest_version("dragoonDorise/EmuDeck")
 
-        cores = self._fetch_installed_emulators()
+        revision = launcher_revision(self.checkbios_url)
+        cores = self._fetch_installed_emulators(revision)
         standalone = [core for core in cores if core != "retroarch"]
-        tree = RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url))
+        tree = RepoTree.github(EMUDECK_REPO, revision)
         libretro = set(installed_libretro_cores(tree.read(RETROARCH_SCRIPT)))
 
         return {
@@ -501,16 +502,16 @@ class Scraper(BaseScraper):
             "launchers": extract_launchers(tree),
         }
 
-    def _fetch_installed_emulators(self) -> list[str]:
-        """Fetch the list of emulators installed by EmuDeck from EmuScripts.
-
-        Returns core names normalized to match emulator profile keys.
+    def _fetch_installed_emulators(self, revision: str) -> list[str]:
+        """The emulators EmuDeck's EmuScripts install at the pinned revision,
+        named as their profiles are: the launcher layer and the SteamOS
+        target are read at that revision too.
         """
         import json
 
         api_url = (
             "https://api.github.com/repos/dragoonDorise/EmuDeck/"
-            "contents/functions/EmuScripts"
+            f"contents/functions/EmuScripts?ref={revision}"
         )
         try:
             req = urllib.request.Request(api_url, headers=github_headers())

scripts/scraper/game_dat_importer.py

@@ -38,7 +38,7 @@ _SCRIPTS_DIR = str(Path(__file__).resolve().parents[1])
 if _SCRIPTS_DIR not in sys.path:
     sys.path.insert(0, _SCRIPTS_DIR)
 
-from artifacts import copy_stream_atomic, write_text_atomic  # noqa: E402
+from artifacts import copy_stream_atomic, write_if_changed, write_text_atomic  # noqa: E402
 from common import parse_untrusted_xml  # noqa: E402
 from region import REGIONS, WORLD  # noqa: E402
 from region_audit import NOINTRO  # noqa: E402
@@ -275,14 +275,6 @@ def dump_catalog(doc: dict) -> str:
     return f'{text},"games":[\n{games}\n]}}\n'
 
 
-def write_if_changed(path: Path, text: str) -> bool:
-    if path.is_file() and path.read_text(encoding="utf-8") == text:
-        return False
-    path.parent.mkdir(parents=True, exist_ok=True)
-    write_text_atomic(str(path), text)
-    return True
-
-
 # Fetching ---------------------------------------------------------------
 
 
@@ -503,7 +495,9 @@ def main() -> int:
             failures.append(f"{doc['source']}/{doc['system']}: {cov['games']} games, "
                             f"floor {floor_of(doc)}")
         if not args.dry_run:
-            write_if_changed(args.output_dir / doc["source"] / f"{doc['system']}.json", text)
+            target = args.output_dir / doc["source"] / f"{doc['system']}.json"
+            target.parent.mkdir(parents=True, exist_ok=True)
+            write_if_changed(str(target), text)
     print(f"{len(docs)} catalogs, {raw_total} bytes raw, {gz_total} bytes gzip")
     for failure in failures:
         print(f"FAIL {failure}", file=sys.stderr)

scripts/scraper/targets/emudeck_targets_scraper.py

@@ -12,7 +12,6 @@ from __future__ import annotations
 
 import argparse
 import json
-import re
 import sys
 import urllib.error
 import urllib.request
@@ -22,6 +21,7 @@ import yaml
 
 from common import github_token, yaml_load
 
+from ..emudeck_launchers import SCRIPT_SUFFIXES, emulator_of_script
 from . import BaseTargetScraper
 
 PLATFORM_NAME = "emudeck"
@@ -33,24 +33,6 @@ WINDOWS_API = (
     "https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts"
 )
 
-# Map EmuDeck script names to emulator profile keys
-# Script naming: emuDeckDolphin.sh -> dolphin
-# Some need explicit mapping when names differ
-_NAME_OVERRIDES: dict[str, str] = {
-    "pcsx2qt": "pcsx2",
-    "rpcs3legacy": "rpcs3",
-    "cemuproton": "cemu",
-    "rmg": "mupen64plus_next",
-    "bigpemu": "bigpemu",
-    "eden": "eden",
-    "suyu": "suyu",
-    "ares": "ares",
-}
-
-# Scripts that are not emulators (config helpers, etc.)
-_SKIP = {"retroarch_maincfg", "retroarch"}
-
-
 def _fetch(url: str) -> str:
     headers = {"User-Agent": "retrobios-scraper/1.0"}
     token = github_token()
@@ -69,26 +51,13 @@ def _list_emuscripts(api_url: str) -> list[str]:
     """List emulator script filenames from GitHub API."""
     entries = json.loads(_fetch(api_url))
     names = [
-        e["name"] for e in entries if e.get("name", "").endswith((".sh", ".ps1"))
+        e["name"] for e in entries if e.get("name", "").endswith(SCRIPT_SUFFIXES)
     ]
     if not names:
         raise RuntimeError(f"no emulator scripts listed at {api_url}")
     return names
 
 
-def _script_to_core(filename: str) -> str | None:
-    """Convert EmuScripts filename to core profile key."""
-    # Strip extension and emuDeck prefix
-    name = re.sub(r"\.(sh|ps1)$", "", filename, flags=re.IGNORECASE)
-    name = re.sub(r"^emuDeck", "", name, flags=re.IGNORECASE)
-    if not name:
-        return None
-    key = name.lower()
-    if key in _SKIP:
-        return None
-    return _NAME_OVERRIDES.get(key, key)
-
-
 class Scraper(BaseTargetScraper):
     """Fetches emulator lists for EmuDeck SteamOS and Windows targets."""
 
@@ -114,14 +83,13 @@ class Scraper(BaseTargetScraper):
         seen: set[str] = set()
         has_retroarch = False
         for script in scripts:
-            core = _script_to_core(script)
-            if core and core not in seen:
+            core = emulator_of_script(script)
+            # RetroArch stands for its libretro cores, added below.
+            if core == "retroarch":
+                has_retroarch = True
+            elif core and core not in seen:
                 seen.add(core)
                 cores.append(core)
-            # Detect RetroArch presence (provides all libretro cores)
-            name = re.sub(r"\.(sh|ps1)$", "", script, flags=re.IGNORECASE)
-            if name.lower() in ("emudeckretroarch", "retroarch_maincfg"):
-                has_retroarch = True
 
         standalone_count = len(cores)
         # EmuDeck ships RetroArch = all its libretro cores are available

scripts/slots.py

@@ -677,12 +677,21 @@ class Collision:
     resolved: list[str]
     # Each declaration with the file it resolved to, in the platform's order.
     declarations: list[tuple[dict, str | None]] = field(default_factory=list)
+    # The system of each declaration, in the same order.
+    systems: list[str] = field(default_factory=list)
+    # The file the pack ships at the destination, as the builder decides it.
+    shipped: str | None = None
 
 
 def _collision_json(collision: Collision) -> dict:
     return {
         "destination": collision.destination,
         "resolved": list(collision.resolved),
+        "shipped": collision.shipped,
+        "declarations": [
+            {"system": system, "name": entry.get("name", ""), "file": local}
+            for system, (entry, local) in zip(collision.systems, collision.declarations)
+        ],
     }
 
 
@@ -702,19 +711,27 @@ def find_collisions(
     CoCo disk ROM at one bios/disk.rom.
     """
     fold = bool(config.get("case_insensitive_fs", False))
-    by_dest: dict[str, list[dict]] = {}
-    for system in (config.get("systems") or {}).values():
+    systems = config.get("systems") or {}
+    # The builder lets a hash-constrained declaration that resolves by
+    # content take the path, else the first it reaches walking the systems
+    # sorted: the same choice says which file ships.
+    preferred = set(
+        preferred_entries(systems, db, "bios", "", False, zip_contents, True).values()
+    )
+    by_dest: dict[str, list[tuple[str, dict]]] = {}
+    for sys_id, system in systems.items():
         for entry in system.get("files") or []:
             if not isinstance(entry, dict):
                 continue
             dest = entry.get("destination") or entry.get("name") or ""
             if dest:
-                by_dest.setdefault(_normalize(dest, fold), []).append(entry)
+                by_dest.setdefault(_normalize(dest, fold), []).append((sys_id, entry))
 
     collisions = []
-    for key, entries in sorted(by_dest.items()):
-        if len(entries) < 2:
+    for key, claims in sorted(by_dest.items()):
+        if len(claims) < 2:
             continue
+        entries = [entry for _sys_id, entry in claims]
         resolved = []
         declarations = []
         for entry in entries:
@@ -734,8 +751,22 @@ def find_collisions(
             # also names the first one's file.
             if any(_accepted_by_all(path, entries, db) for path in resolved):
                 continue
+            reached = sorted(
+                (sys_id, local)
+                for (sys_id, _entry), (_e, local) in zip(claims, declarations) if local
+            )
+            shipped = next(
+                (local for entry, local in declarations if local and id(entry) in preferred),
+                reached[0][1],
+            )
             collisions.append(
-                Collision(destination=key, resolved=resolved, declarations=declarations)
+                Collision(
+                    destination=key,
+                    resolved=resolved,
+                    declarations=declarations,
+                    systems=[sys_id for sys_id, _entry in claims],
+                    shipped=shipped,
+                )
             )
     return collisions
 
@@ -863,11 +894,18 @@ def _same_file_family(paths: list[str]) -> bool:
 
 
 def format_collision(collision: Collision) -> str:
-    """One line naming the destination and the files fighting for it."""
-    return (
-        f"{collision.destination}: declared for "
-        + " and for ".join(collision.resolved)
-    )
+    """One line naming the destination, which system declares which file,
+    and the one the pack ships there."""
+    owners: dict[str, list[str]] = {}
+    for system, (_entry, local) in zip(collision.systems, collision.declarations):
+        if local and system not in owners.setdefault(local, []):
+            owners[local].append(system)
+    parts = [
+        f"{path} ({', '.join(owners[path])}) "
+        + ("packed" if path == collision.shipped else "left out")
+        for path in collision.resolved
+    ]
+    return f"{collision.destination}: " + ", ".join(parts)
 
 
 def format_conflict(conflict: Conflict) -> str:

scripts/split_pack.py

@@ -338,6 +338,10 @@ def main() -> int:
     # The packs are rewritten in place by generate_pack under this lock: a
     # build running beside the split mixes its bytes into the pack being
     # read, or writes a pack that the split then deletes.
+    if not args.target.exists():
+        print(f"Error: {args.target}: no such pack or directory, pass a pack or the "
+              "--output-dir generate_pack wrote (default dist/)", file=sys.stderr)
+        return 1
     directory = args.target if args.target.is_dir() else args.target.parent
     try:
         with artifact_lock(str(directory)):

scripts/validate_pr.py

@@ -18,6 +18,7 @@ Outputs a structured report suitable for PR comments.
 from __future__ import annotations
 
 import argparse
+import hashlib
 import json
 import os
 import re
@@ -25,6 +26,7 @@ import shutil
 import subprocess
 import sys
 import urllib.parse
+import zipfile
 from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
@@ -335,6 +337,53 @@ def load_profile_hashes(
     return known
 
 
+def _archive_members(
+    filepath: str, db: dict | None, platform_hashes: dict, profile_hashes: dict
+) -> list[str] | None:
+    """What each file of a ZIP is, None when filepath is not a ZIP.
+
+    Read only for an archive no list declares as a whole: MAME and FBNeo
+    sets are ZIP BIOS files in their own right.
+    """
+    if not zipfile.is_zipfile(filepath):
+        return None
+    held = (db or {}).get("files", {})
+    described = []
+    try:
+        with zipfile.ZipFile(filepath) as archive:
+            for info in archive.infolist():
+                if info.is_dir():
+                    continue
+                if info.file_size > MAX_FILE_SIZE:
+                    described.append(f"`{info.filename}` too large to read here")
+                    continue
+                sha1, md5 = hashlib.sha1(), hashlib.md5()
+                with archive.open(info) as handle:
+                    for chunk in iter(lambda: handle.read(1 << 20), b""):
+                        sha1.update(chunk)
+                        md5.update(chunk)
+                sha1_hex, md5_hex = sha1.hexdigest(), md5.hexdigest()
+                crc = f"{info.CRC:08x}"
+                if sha1_hex in held:
+                    state = f"already held at `{held[sha1_hex]['path']}`"
+                elif (
+                    sha1_hex in platform_hashes.get("sha1", set())
+                    or md5_hex in platform_hashes.get("md5", set())
+                    or (crc, info.file_size) in platform_hashes.get("crc32", set())
+                    or profile_hashes.get("sha1", {}).get(sha1_hex)
+                    or profile_hashes.get("md5", {}).get(md5_hex)
+                    or profile_hashes.get("crc32", {}).get((crc, info.file_size))
+                ):
+                    state = "declared and not held"
+                else:
+                    state = "not recognised"
+                described.append(f"`{info.filename}` {state}")
+    except (zipfile.BadZipFile, RuntimeError, OSError) as exc:
+        # An encrypted member raises RuntimeError.
+        described.append(f"unreadable ({exc})")
+    return described
+
+
 def validate_file(
     filepath: str,
     db: dict | None,
@@ -446,6 +495,8 @@ def validate_file(
         or profile_hashes.get("crc32", {}).get((result.crc32, None))
     )
 
+    # The files of a ZIP no list declares whole, when the upload is one.
+    members: list[str] | None = None
     if sha1_known:
         result.add_check(True, "SHA1 matches known platform requirement")
     elif md5_known:
@@ -466,11 +517,26 @@ def validate_file(
         )
     else:
         result.outcome = "unknown"
-        result.add_warning(
-            "Not recognised: no platform list or emulator profile declares this "
-            f"file. Say in the {'issue' if too_large else 'pull request'} which "
-            "emulator reads it, for which system, and where the dump comes from"
+        members = None if too_large else _archive_members(
+            filepath, db, platform_hashes, profile_hashes
         )
+        if members is not None:
+            # A folder zipped to upload many files at once was hashed as one
+            # file and its contributor asked where that dump came from.
+            shown = "; ".join(members[:10]) + (
+                f"; and {len(members) - 10} more" if len(members) > 10 else ""
+            )
+            result.add_warning(
+                "A ZIP no platform list or emulator profile declares as a whole: "
+                "add the files it holds, unzipped, not the archive. Inside: "
+                + (shown or "nothing")
+            )
+        else:
+            result.add_warning(
+                "Not recognised: no platform list or emulator profile declares this "
+                f"file. Say in the {'issue' if too_large else 'pull request'} which "
+                "emulator reads it, for which system, and where the dump comes from"
+            )
 
     normalized = os.path.normpath(relative)
     # A platform's declaration first, then a profile's: most files the gaps
@@ -503,7 +569,8 @@ def validate_file(
             result.add_check(True, f"Correct placement: bios/{parts[1]}/{parts[2]}/")
         else:
             result.add_warning("File should be in bios/Manufacturer/Console/ structure")
-    elif result.outcome == "held":
+    elif result.outcome == "held" or members is not None:
+        # Nothing to place: a copy, or an archive whose files go in unzipped.
         pass
     elif expected_folders:
         # The contributor asks before forking: say where it goes.
@@ -847,7 +914,8 @@ def main() -> None:
                 {
                     "file": r.filepath,
                     "passed": r.passed,
-                    "outcome": r.outcome or "new",
+                    # A file that failed, unread or refused, is not a new one.
+                    "outcome": r.outcome or ("new" if r.passed else "failed"),
                     "destination": r.destination,
                     "sha1": r.sha1,
                     "md5": r.md5,

scripts/validate_schemas.py

@@ -14,7 +14,15 @@ from pathlib import Path, PurePosixPath
 import yaml
 
 from common import source_yaml_paths, yaml_load
-from jsonschema import Draft202012Validator, FormatChecker
+
+try:
+    from jsonschema import Draft202012Validator, FormatChecker
+except ImportError:
+    # Tests import this module and skip on the ImportError, the command
+    # names the install pipeline.py names for the same step.
+    if __name__ != "__main__":
+        raise
+    sys.exit('Error: jsonschema required (pip install "jsonschema[format-nongpl]")')
 
 ROOT = Path(__file__).resolve().parent.parent
 SCHEMAS = ROOT / "schemas"

scripts/verify.py

@@ -2988,13 +2988,13 @@ def _main(held: contextlib.ExitStack) -> None:
             print(_overall_summary(group_results))
 
     if args.destinations:
-        held = {}
+        destinations = {}
         for (result, names), (group, _rep) in zip(group_results, groups):
             for name, key in zip(names, group):
-                held[name] = {
+                destinations[name] = {
                     "platform": key, "pack_destinations": result["pack_destinations"],
                 }
-        Path(args.destinations).write_text(json.dumps(held, indent=1, sort_keys=True))
+        Path(args.destinations).write_text(json.dumps(destinations, indent=1, sort_keys=True))
 
     if args.json:
         for r in all_results.values():