chore: merge scraper and tool fixes

scripts/check_freshness.py

@@ -54,11 +54,13 @@ import pin_installer  # noqa: E402
 import refresh_data_dirs  # noqa: E402
 import upstream  # noqa: E402
 from common import (  # noqa: E402
+    InputMissing,
     apply_target_overrides,
     list_registered_platforms,
     load_emulator_profiles,
     load_platform_registry,
     require_directory,
+    suggest_names,
     upstream_profile_index,
     write_text_atomic,
     yaml_load,
@@ -73,9 +75,13 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
 CORE_INFO_REPO = "https://github.com/libretro/libretro-core-info"
 # The git blob of every .info as last reviewed: `--record-coreinfo` writes it.
 CORE_INFO_RECORD = REPO_ROOT / "platforms" / "_core_info.json"
-NO_INTRO_MIRROR = "https://github.com/hugo19941994/auto-datfile-generator"
-NO_INTRO_RELEASE = "Daily_Rebuild"
-NO_INTRO_ASSET = "no-intro.zip"
+NO_INTRO_MIRROR = game_dat_importer.NOINTRO_MIRROR
+NO_INTRO_RELEASE = game_dat_importer.NOINTRO_RELEASE
+NO_INTRO_ASSET = game_dat_importer.NOINTRO_ASSET
+# The mirror rebuilds daily. A pack older than this is a mirror that stopped:
+# hugo19941994's failed every day from 2026-07-08, and comparing our import
+# date with its asset read OK for four months.
+NO_INTRO_MAX_AGE_DAYS = 7
 TOSEC_DOWNLOADS = "https://www.tosecdev.org/downloads"
 MAME_REPO = "https://github.com/mamedev/mame"
 FBNEO_REPO = "https://github.com/libretro/FBNeo"
@@ -893,7 +899,10 @@ def record_core_info(
     """
     unknown = sorted(set(names) - set(blobs))
     if unknown:
-        raise ValueError(f"no such .info in core-info: {', '.join(unknown)}")
+        raise InputMissing(
+            f"no such .info in core-info: {', '.join(unknown)}. "
+            + suggest_names(unknown, list(blobs), "libretro-core-info")
+        )
     recorded = load_core_info_record(path) or {}
     recorded.update({n: blobs[n] for n in (names or blobs)})
     payload = {"repository": CORE_INFO_REPO, "blobs": dict(sorted(recorded.items()))}
@@ -1083,6 +1092,25 @@ def _no_intro_asset_date(cache_dir: str, offline: bool) -> str:
     return ""
 
 
+def _today() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+
+def _no_intro_row(subject: str, local: str, updated: str) -> Finding:
+    """Our import against the mirror's pack, and the pack against today."""
+    age = (datetime.fromisoformat(_today()) - datetime.fromisoformat(updated)).days
+    if age > NO_INTRO_MAX_AGE_DAYS:
+        return Finding(
+            "catalogs", subject, STALE, local=local, remote=updated,
+            detail=f"the mirror last rebuilt {NO_INTRO_ASSET} on {updated}, {age} days ago, "
+                   f"though it rebuilds daily: {NO_INTRO_MIRROR}/actions",
+        )
+    return Finding(
+        "catalogs", subject, STALE if updated > local else OK,
+        local=local, remote=updated, detail=f"{NO_INTRO_ASSET} rebuilt {updated}",
+    )
+
+
 def _check_nointro_games(cache_dir: str, offline: bool) -> Finding:
     """games/catalogs/no-intro/ against the pack it was read from. The pack
     holds every DAT: rebuilt after the catalogs were read, it may carry
@@ -1094,10 +1122,7 @@ def _check_nointro_games(cache_dir: str, offline: bool) -> Finding:
     updated = _no_intro_asset_date(cache_dir, offline)
     if not updated:
         return Finding("catalogs", subject, UNKNOWN, local=fetched, detail="asset not listed")
-    return Finding(
-        "catalogs", subject, STALE if updated > fetched else OK,
-        local=fetched, remote=updated, detail=f"{NO_INTRO_ASSET} rebuilt {updated}",
-    )
+    return _no_intro_row(subject, fetched, updated)
 
 
 def redump_datfile_version(code: str) -> str:
@@ -1171,14 +1196,7 @@ def _check_no_intro(cache_dir: str, offline: bool) -> Finding:
     updated = _no_intro_asset_date(cache_dir, offline)
     if not updated:
         return Finding("catalogs", "no-intro", UNKNOWN, local=imported, detail="asset not listed")
-    return Finding(
-        "catalogs",
-        "no-intro",
-        STALE if updated > imported else OK,
-        local=imported,
-        remote=updated,
-        detail=f"{NO_INTRO_ASSET} rebuilt {updated}",
-    )
+    return _no_intro_row("no-intro", imported, updated)
 
 
 def tosec_latest_pack(html: str) -> str | None:
@@ -1839,7 +1857,10 @@ def main() -> int:
     if args.record_coreinfo is not None:
         blobs = core_info_blobs(args.cache_dir, args.offline)
         if not blobs:
-            raise RuntimeError("core-info listing unavailable: nothing recorded")
+            raise InputMissing(
+                f"core-info listing not in {args.cache_dir}: run without --offline"
+                if args.offline else "core-info listing unavailable: the GitHub API did not answer"
+            )
         done = record_core_info(blobs, args.record_coreinfo)
         print(f"recorded {len(done)} .info file(s) as reviewed")
         return 0

scripts/check_platform_folder.py

@@ -1,11 +1,9 @@
-"""Check a BIOS folder against one platform's list, from the published data only:
-python check_platform_folder.py batocera path/to/bios. Exit 1 on a file wrong or missing."""
+"""Usage: python check_platform_folder.py batocera path/to/bios. Exit 1 on a file wrong or missing."""
 from __future__ import annotations
 
 import hashlib
 import json
 import sys
-import urllib.error
 import urllib.request
 import zipfile
 from pathlib import Path
@@ -42,14 +40,12 @@ def check(platform: str, root: Path, api: str = API) -> list[tuple[str, str]]:
     """(destination, ok | wrong | missing) per entry. With no destination (EmuDeck), any file with its hash passes."""
     with urllib.request.urlopen(f"{api}{platform}.json") as response:
         contract = json.load(response)["items"][0]["contract"]
-    mode, results = contract["verification_mode"], []
-    loose = [path for path in root.iterdir() if path.is_file()] if root.is_dir() else []
+    mode, results, loose = contract["verification_mode"], [], [p for p in root.iterdir() if p.is_file()]
     for system_id, system in contract["systems"].items():
         listed = [e for e in system.get("files") or [] if not e.get("destination")]
         for entry in (e for e in system.get("files") or [] if e.get("destination")):
             path = root / entry["destination"]
-            verdict = "ok" if accepts(entry, path, mode) else "wrong" if path.is_file() else "missing"
-            results.append((entry["destination"], verdict))
+            results.append((entry["destination"], "ok" if accepts(entry, path, mode) else "wrong" if path.is_file() else "missing"))
         if listed:
             held = any(accepts(e, path, mode) for e in listed for path in loose)
             results.append((f"{system_id}: any file", "ok" if held else "missing"))
@@ -57,12 +53,16 @@ def check(platform: str, root: Path, api: str = API) -> list[tuple[str, str]]:
 
 
 if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        sys.exit(__doc__)
     try:
         found = check(sys.argv[1], Path(sys.argv[2]))
-    except urllib.error.HTTPError as error:  # a 404: no platform has that name
+    except urllib.request.HTTPError as error:  # a 404: no platform has that name
         with urllib.request.urlopen(API.replace("platforms/", "catalog.json")) as response:
             known = sorted(i["url"][10:-5] for i in json.load(response)["items"] if i["url"].startswith("manifests/"))
         sys.exit(f"unknown platform: {sys.argv[1]}. Known: {', '.join(known)}" if error.code == 404 else error)
+    except OSError as error:  # no such folder, or no network (URLError)
+        sys.exit(f"error: {error}")
     for verdict in ("ok", "wrong", "missing"):
         names = sorted({dest for dest, seen in found if seen == verdict})
         print(f"{verdict}: {len(names)}" + "".join(f"\n  {name}" for name in names if verdict != "ok"))

scripts/exporter/recalbox_exporter.py

@@ -221,12 +221,19 @@ class Exporter(BaseExporter):
         ours = str(entry.get("path") or entry.get("name", "")).split("/")[:-1]
         return not {d.casefold() for d in theirs} & {d.casefold() for d in ours}
 
-    def _bios_element(self, fe: NativeFile, native_id: str) -> str:
+    def _bios_element(
+        self, fe: NativeFile, native_id: str, also: tuple[NativeFile, ...] = ()
+    ) -> str:
+        """``also`` are further additions at fe's path: Bios.cpp reads one
+        element per path, and its md5 and core lists take every one of them."""
+        group = (fe, *also)
+        md5 = dict.fromkeys(h for f in group for h in f.hashes("md5"))
+        cores = dict.fromkeys(c for f in group for c in f.cores())
         attrs = [f"path={quoteattr(self._path(fe))}"]
-        attrs.append(f'md5={quoteattr(",".join(fe.hashes("md5")))}')
-        attrs.append(f'core={quoteattr(",".join(fe.cores()))}')
+        attrs.append(f'md5={quoteattr(",".join(md5))}')
+        attrs.append(f'core={quoteattr(",".join(cores))}')
 
-        if not self.required_value(fe):
+        if not any(self.required_value(f) for f in group):
             attrs.append('mandatory="false"')
         elif fe.native("mandatory_declared", None) is True:
             attrs.append('mandatory="true"')
@@ -343,11 +350,17 @@ class Exporter(BaseExporter):
         }
         ours: dict[str, tuple[NativeSystem, dict[str, tuple[NativeFile, str]]]] = {}
         for system in systems.values():
-            writable = [fe for fe in system.files if self.writable(fe)]
-            if writable:
+            # Two additions at one path (two cores, two dumps) are one
+            # element: keyed by path, the second replaced the first and its
+            # md5 left the file while both were counted added.
+            at_path: dict[str, list[NativeFile]] = {}
+            for fe in system.files:
+                if self.writable(fe):
+                    at_path.setdefault(self._path(fe), []).append(fe)
+            if at_path:
                 ours[system.native_id] = (system, {
-                    self._path(fe): (fe, self._bios_element(fe, system.native_id))
-                    for fe in writable
+                    path: (group[0], self._bios_element(group[0], system.native_id, tuple(group[1:])))
+                    for path, group in at_path.items()
                 })
 
         def element(attrs: dict[str, str]) -> str:

scripts/exporter/retrodeck_exporter.py

@@ -215,6 +215,7 @@ class Exporter(BaseExporter):
         ours: list[OrderedDict],
         additions: list[OrderedDict] | None = None,
         appended: set[tuple[str, str]] | None = None,
+        held_back: list[int] | None = None,
     ) -> list[OrderedDict]:
         """Correct the component's own list, never replace it.
 
@@ -301,13 +302,19 @@ class Exporter(BaseExporter):
             for e in merged
             for system in (_systems_of(e) or [""])
         }
-        for entry in additions or []:
+        for index, entry in enumerate(additions or []):
             key = (str(entry.get("filename", "")), str(entry.get("system", "")), _paths_key(entry))
             if key[0] and key not in held:
                 held.add(key)
                 merged.append(entry)
+            elif held_back is not None:
+                held_back.append(index)
         return merged
 
+    def lands(self, fe: NativeFile) -> bool:
+        """An addition the list already held is not written a second time."""
+        return self.writable(fe) and id(fe) not in getattr(self, "_held_back", set())
+
     def render(
         self,
         systems: dict[str, NativeSystem],
@@ -317,6 +324,7 @@ class Exporter(BaseExporter):
     ) -> dict[str, str]:
         self._place_additions(systems)
         self._appended: set[int] = set()
+        self._held_back: set[int] = set()
         grouped = self._by_component(systems)
         produced: dict[str, str] = {}
 
@@ -331,25 +339,28 @@ class Exporter(BaseExporter):
                 continue
 
             entries = [self._entry(fe) for fe in files if fe.platform is not None]
-            additions = [self._entry(fe) for fe in files if fe.platform is None]
+            added = [fe for fe in files if fe.platform is None]
+            additions = [self._entry(fe) for fe in added]
             appended: set[tuple[str, str]] = set()
+            held_back: list[int] = []
             for component_value in manifest.values():
                 if not isinstance(component_value, dict):
                     continue
                 holder = self._bios_holder(component_value)
                 if holder is None:
                     component_value["bios"] = self._merge(
-                        None, entries, additions, appended
+                        None, entries, additions, appended, held_back
                     )
                 else:
                     container, key = holder
                     written = container.get(key)
-                    merged = self._merge(written, entries, additions, appended)
+                    merged = self._merge(written, entries, additions, appended, held_back)
                     # A single object stays one while nothing is added to it.
                     container[key] = (
                         merged[0] if isinstance(written, dict) and len(merged) == 1 else merged
                     )
                 break
+            self._held_back.update(id(added[index]) for index in held_back)
             # A declared entry whose revision the platform does not hold is
             # written beside theirs: a new line, not a corrected value.
             self._appended.update(

scripts/exporter/retropie_exporter.py

@@ -336,6 +336,8 @@ class Exporter(BaseExporter):
                     systems_of.setdefault(str(name).lower(), set()).add(fe.native_system)
         known = set(systems_of)
         self._skipped: dict[str, list[str]] = {}
+        # Rewritten, with a file of a system no clause lists left out.
+        self._partial: list[str] = []
         self._named: set[int] = set()
         produced: dict[str, str] = {}
 
@@ -400,12 +402,11 @@ class Exporter(BaseExporter):
                     homes.setdefault(owners[0], []).append(fe)
                 else:
                     homeless = True
-            if homeless:
-                # No enumeration of that system to extend, and a sentence we
-                # would have to write ourselves is a documentation change,
-                # not a correction.
-                skip("whose help text lists no files of that system to add to", module_id)
+            # No enumeration of that system to extend, and a sentence we
+            # would have to write ourselves is a documentation change, not a
+            # correction. With nothing placed the package is left as is.
             if not homes:
+                skip("whose help text lists no files of that system to add to", module_id)
                 continue
 
             added = sum(len(self._in_search_order(fes)) for fes in homes.values())
@@ -428,6 +429,8 @@ class Exporter(BaseExporter):
                 f'rp_module_help="{new_help}"',
                 1,
             )
+            if homeless:
+                self._partial.append(module_id)
 
         return produced
 
@@ -454,12 +457,20 @@ class Exporter(BaseExporter):
     ) -> str:
         """Packages are the unit here, not file entries."""
         skipped: dict[str, list[str]] = getattr(self, "_skipped", {})
+
+        def shown(modules: list[str]) -> str:
+            names = ", ".join(sorted(modules)[:4])
+            return names + (f" and {len(modules) - 4} more" if len(modules) > 4 else "")
+
         parts = [f"{len(produced)} packages updated"]
+        partial = getattr(self, "_partial", [])
+        if partial:
+            parts.append(
+                f"Updated in part: {shown(partial)}, whose help text lists no files "
+                "of one of their systems to add to"
+            )
         for reason, modules in sorted(skipped.items()):
-            shown = ", ".join(sorted(modules)[:4])
-            if len(modules) > 4:
-                shown += f" and {len(modules) - 4} more"
-            parts.append(f"Left as is: {shown}, {reason}")
+            parts.append(f"Left as is: {shown(modules)}, {reason}")
         return ". ".join(parts)
 
     def validate(

scripts/exporter/systemdat_exporter.py

@@ -175,6 +175,9 @@ class Exporter(BaseExporter):
 
         ours: dict[str, dict[str, str]] = {}
         added: dict[str, set[str]] = {}
+        # A rom name is one line: an addition whose name an earlier entry
+        # took is not written, and was counted added.
+        self._shadowed: set[int] = set()
         for system, files in self.exportable(systems):
             rendered: dict[str, str] = {}
             for fe in files:
@@ -194,7 +197,11 @@ class Exporter(BaseExporter):
                 sha1 = fe.record_hash("sha1")
                 if sha1:
                     parts.append(f"sha1 {sha1}")
-                rendered.setdefault(name, f"\trom ( {' '.join(parts)} )")
+                if name in rendered:
+                    if fe.platform is None:
+                        self._shadowed.add(id(fe))
+                    continue
+                rendered[name] = f"\trom ( {' '.join(parts)} )"
                 if fe.platform is None:
                     added.setdefault(system.native_id, set()).add(name)
             if rendered:
@@ -241,7 +248,11 @@ class Exporter(BaseExporter):
         return {self.native_filename(): "\n".join(lines)}
 
     def lands(self, fe: NativeFile) -> bool:
-        return self.writable(fe) and id(fe) not in getattr(self, "_untitled", set())
+        return (
+            self.writable(fe)
+            and id(fe) not in getattr(self, "_untitled", set())
+            and id(fe) not in getattr(self, "_shadowed", set())
+        )
 
     def validate(
         self,

scripts/generate_site.py

@@ -55,6 +55,7 @@ from common import (
     load_platform_config,
     load_provenance_snapshots,
     native_layer,
+    require_directory,
     require_yaml,
     resolution_is_hash_exact,
     resolve_local_file,
@@ -2330,7 +2331,7 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N
         f"({REPO_URL}/blob/main/scripts/check_platform_folder.py). A test runs it "
         "against the verifier on a sample folder, so the two give the same answer. "
         "It exits 1 when a file is wrong or missing. For an unknown platform it "
-        "prints the known ones.",
+        "prints the known ones, and it names a folder that does not exist.",
         "",
         "`install.py --check` answers another question by default: whether the folder "
         "holds what the pack installs, the manifest above. With `--declared-only`, or "
@@ -12390,6 +12391,10 @@ def main() -> None:
     args = parser.parse_args()
     if args.docs_dir is None:
         args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT)
+    # Before any write: the public assets and the bootstraps went out first,
+    # and a mistyped folder stopped the run only after them.
+    require_directory(args.emulators_dir, "--emulators-dir")
+    require_directory(args.platforms_dir, "--platforms-dir")
     try:
         # Two runs over one docs/ delete each other's pages and scratch
         # files in the sweep, so the second is refused instead.

scripts/profile_sync.py

@@ -20,6 +20,7 @@ import re
 import subprocess
 import sys
 import tempfile
+import time
 from collections import Counter
 from collections.abc import Callable, Iterable, Iterator, Sequence
 from dataclasses import asdict, dataclass, field, replace
@@ -80,6 +81,9 @@ from upstream import CompareResult, find_renamed
 DEFAULT_CACHE = upstream.CACHE_DIR
 ANON_QUOTA = 60
 TRIAGE_PATH_SAMPLE = 5
+# A pass that asks the forges says where it is at this interval: an online
+# --all --triage ran 15 minutes without a word and read as hung.
+PROGRESS_SECONDS = 30
 
 STATUS_ORDER = (
     "ANCHORED", "UNCHECKED", "EXTERNAL", "BINARY", "SHIFTED", "RENAMED", "MOVED",
@@ -4991,35 +4995,40 @@ def _print_extras_from(
             print(f"  {line}")
 
 
+class _Progress:
+    """A stderr line every PROGRESS_SECONDS while a pass asks the forges."""
+
+    def __init__(self, total: int, doing: str, quiet: bool) -> None:
+        self.total, self.doing, self.quiet = total, doing, quiet
+        self.last = time.monotonic()
+
+    def step(self, done: int, name: str) -> None:
+        now = time.monotonic()
+        if not self.quiet and now - self.last >= PROGRESS_SECONDS:
+            print(f"{self.doing}: {done} of {self.total}, at {name}", file=sys.stderr, flush=True)
+            self.last = now
+
+
+def _interrupted(done: int, total: int, what: str) -> None:
+    print(f"interrupted after {done} of {total} profiles: {what}", file=sys.stderr, flush=True)
+
+
 def _print_triage(
     args: argparse.Namespace, selected: dict, reports: list[ProfileReport]
-) -> None:
-    """Profiles ranked by drift, worst first."""
+) -> bool:
+    """Profiles ranked by drift, worst first. False when Ctrl-C cut the
+    scoring short: the ranking then covers the profiles scored."""
     ranked = []
     sampled = 0
-    for report in reports:
-        profile = selected[report.name]
-        version = None
-        commits = 0
-        refused = ""
-        repo = select_repo(profile)
-        if report.head and repo is not None:
-            paths = _cited_paths(report)
-            try:
-                version = check_version(profile, repo, args.cache_dir, args.offline)
-                for path in paths[:TRIAGE_PATH_SAMPLE]:
-                    commits += upstream.commits_touching(
-                        repo, path, report.pin, args.cache_dir, args.offline,
-                        head=report.head,
-                    )
-            except upstream.UpstreamError as exc:
-                # One forge refusing a request (Eden's answers 403 to commit
-                # lookups) leaves this profile unscored, not the whole ranking.
-                refused = "commit history unreachable"
-                print(f"{report.name}: {exc}", file=sys.stderr)
-            if len(paths) > TRIAGE_PATH_SAMPLE and not refused:
-                sampled += 1
-        ranked.append((drift_score(report, version, commits), report, refused))
+    progress = _Progress(len(reports), "scored", args.offline or len(reports) < 2)
+    complete = True
+    try:
+        for done, report in enumerate(reports):
+            progress.step(done, report.name)
+            sampled += _score(args, selected, report, ranked)
+    except KeyboardInterrupt:
+        _interrupted(len(ranked), len(reports), "the ranking covers those scored")
+        complete = False
     if sampled:
         # One line for the run: a notice per profile buried the ranking
         # under 254 lines of stderr.
@@ -5037,6 +5046,37 @@ def _print_triage(
         pending += bool(report.needs_review() or report.uncheckable())
         print(f"{score:5d}  {report.name:30s}  {state}")
     print(f"{len(ranked)} profiles, {pending} with something to review")
+    return complete
+
+
+def _score(
+    args: argparse.Namespace, selected: dict, report: ProfileReport, ranked: list
+) -> bool:
+    """Append one profile's drift score to ``ranked``. True when its commit
+    count was read on a sample of the paths it cites."""
+    profile = selected[report.name]
+    version = None
+    commits = 0
+    refused = ""
+    sampled = False
+    repo = select_repo(profile)
+    if report.head and repo is not None:
+        paths = _cited_paths(report)
+        try:
+            version = check_version(profile, repo, args.cache_dir, args.offline)
+            for path in paths[:TRIAGE_PATH_SAMPLE]:
+                commits += upstream.commits_touching(
+                    repo, path, report.pin, args.cache_dir, args.offline,
+                    head=report.head,
+                )
+        except upstream.UpstreamError as exc:
+            # One forge refusing a request (Eden's answers 403 to commit
+            # lookups) leaves this profile unscored, not the whole ranking.
+            refused = "commit history unreachable"
+            print(f"{report.name}: {exc}", file=sys.stderr)
+        sampled = len(paths) > TRIAGE_PATH_SAMPLE and not refused
+    ranked.append((drift_score(report, version, commits), report, refused))
+    return sampled
 
 
 def _refuse_flag_combinations(args: argparse.Namespace) -> None:
@@ -5260,30 +5300,24 @@ def main() -> None:
         return
 
     reports = []
-    for name, profile in selected.items():
-        profile_path = Path(args.emulators_dir) / f"{name}.yml"
-        try:
-            report = _build_full_report(args, name, profile, profile_path, failed)
-        except upstream.RateLimitError as exc:
-            # A quota is one forge's: git.libretro.com answering 429 said
-            # nothing about the 600 profiles on GitHub. upstream stops asking
-            # that host, so the rest of the pass costs it nothing.
-            print(f"{name}: {exc}", file=sys.stderr)
-            report = ProfileReport(
-                name=name, entries=[], counts={},
-                skipped=f"rate limited: {exc.host or exc}",
-            )
-            failed.add(name)
-            reports.append(report)
-            continue
-        reports.append(report)
-        if writes:
-            try:
-                _apply_writes(args, name, profile, report)
-            except YamlWriteError as exc:
-                # The guard already left the file untouched, so keep going.
-                print(f"{name}: write refused: {exc}", file=sys.stderr)
-                failed.add(name)
+    quiet = args.offline or len(selected) < 2
+    if not quiet:
+        print(
+            f"reading {len(selected)} profiles against their forges, a line every "
+            f"{PROGRESS_SECONDS} s. --offline answers from the cache alone.",
+            file=sys.stderr, flush=True,
+        )
+    progress = _Progress(len(selected), "read", quiet)
+    complete = True
+    try:
+        for name, profile in selected.items():
+            progress.step(len(reports), name)
+            _read_one(args, name, profile, reports, failed, writes)
+    except KeyboardInterrupt:
+        # What was read is still worth printing: a ranking of 200 profiles
+        # beats none after 15 minutes.
+        _interrupted(len(reports), len(selected), "the output covers those read")
+        complete = False
 
     # --changed-only narrows every output, not the text report alone: the
     # documented `--all --changed-only --json` listed every profile.
@@ -5291,14 +5325,49 @@ def main() -> None:
         report for report in reports
         if not args.changed_only or report.needs_review() or report.name in failed
     ]
-    _emit(args, selected, reports, shown)
+    complete = _emit(args, selected, reports, shown) is not False and complete
     _refresh_source_paths(args)
+    if not complete:
+        raise SystemExit(130)
     if failed or (args.strict and any(
         report.needs_review() or report.uncheckable() for report in reports
     )):
         raise SystemExit(1)
 
 
+def _read_one(
+    args: argparse.Namespace,
+    name: str,
+    profile: dict,
+    reports: list[ProfileReport],
+    failed: set[str],
+    writes: bool,
+) -> None:
+    """Build one profile's report into ``reports``, then apply the writes."""
+    profile_path = Path(args.emulators_dir) / f"{name}.yml"
+    try:
+        report = _build_full_report(args, name, profile, profile_path, failed)
+    except upstream.RateLimitError as exc:
+        # A quota is one forge's: git.libretro.com answering 429 said
+        # nothing about the 600 profiles on GitHub. upstream stops asking
+        # that host, so the rest of the pass costs it nothing.
+        print(f"{name}: {exc}", file=sys.stderr)
+        reports.append(ProfileReport(
+            name=name, entries=[], counts={},
+            skipped=f"rate limited: {exc.host or exc}",
+        ))
+        failed.add(name)
+        return
+    reports.append(report)
+    if writes:
+        try:
+            _apply_writes(args, name, profile, report)
+        except YamlWriteError as exc:
+            # The guard already left the file untouched, so keep going.
+            print(f"{name}: write refused: {exc}", file=sys.stderr)
+            failed.add(name)
+
+
 # Profiles this run wrote. Their new pins name trees the site's record of
 # cited source paths does not hold yet.
 _WRITTEN: set[str] = set()
@@ -5400,8 +5469,9 @@ def _emit(
     selected: dict,
     reports: list[ProfileReport],
     shown: list[ProfileReport],
-) -> None:
-    """Write the reports in the form the flags ask for."""
+) -> bool | None:
+    """Write the reports in the form the flags ask for. False when the
+    triage was cut short."""
     if args.as_json:
         print(json.dumps([report_to_dict(r) for r in shown], indent=2))
         return
@@ -5417,8 +5487,7 @@ def _emit(
                 print(url)
         return
     if args.triage:
-        _print_triage(args, selected, shown)
-        return
+        return _print_triage(args, selected, shown)
     for report in shown:
         print(format_report(report, args.changed_only))
         _print_extras(args, selected[report.name], report)

scripts/refresh_stale.py

@@ -148,18 +148,28 @@ def _run_check_freshness(areas: tuple[str, ...], extra: list[str]) -> list[dict]
     if areas and set(areas) != set(AUTO_AREAS + MANUAL_AREAS):
         cmd += ["--only", ",".join(areas)]
     cmd += extra
-    proc = subprocess.run(
-        cmd, cwd=REPO_ROOT, capture_output=True, text=True, check=False, timeout=1800
-    )
-    if proc.returncode not in (0, 1):
+    try:
+        proc = subprocess.run(
+            cmd, cwd=REPO_ROOT, capture_output=True, text=True, check=False, timeout=1800
+        )
+    except subprocess.TimeoutExpired as exc:
+        raise RuntimeError(
+            f"check_freshness.py did not finish in {exc.timeout:.0f} s: run "
+            f"`{' '.join(cmd[1:])} > report.json` alone, then pass --findings report.json"
+        ) from exc
+    # Exit 1 is the STALE rows, and also a crash: only a JSON list on stdout
+    # tells them apart, and the crash's own message is on stderr.
+    try:
+        findings = json.loads(proc.stdout) if proc.returncode in (0, 1) else None
+    except json.JSONDecodeError:
+        findings = None
+    if not isinstance(findings, list):
         tail = (proc.stderr or proc.stdout).strip().splitlines()[-3:]
         raise RuntimeError(
-            "check_freshness.py failed:\n" + "\n".join(tail or ["(no output)"])
+            f"check_freshness.py failed (exit {proc.returncode}):\n"
+            + "\n".join(tail or ["(no output)"])
         )
-    try:
-        return json.loads(proc.stdout)
-    except json.JSONDecodeError as exc:
-        raise RuntimeError(f"check_freshness.py returned unreadable JSON: {exc}") from exc
+    return findings
 
 
 def load_findings(path: Path, areas: tuple[str, ...]) -> list[dict]:

scripts/scraper/arcade_parent_importer.py

@@ -1122,7 +1122,7 @@ def main() -> int:
     which.add_argument("--all", action="store_true", help="every profile with an arcade source")
     parser.add_argument("--emulators-dir", default="emulators",
                         help="profiles whose BIOS sets are read (default emulators)")
-    parser.add_argument("--database", default="database.json",
+    parser.add_argument("--db", default="database.json",
                         help="collection index, to name BIOS sets the collection lacks")
     parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT,
                         help=f"where the parent maps are written (default {DEFAULT_OUTPUT})")
@@ -1133,7 +1133,7 @@ def main() -> int:
     args = parser.parse_args()
 
     profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
-    names = collection_names(Path(args.database))
+    names = collection_names(Path(args.db))
     failed = False
     for profile in sorted(PROFILES) if args.all else args.profile:
         if profile not in profiles:

scripts/scraper/game_dat_importer.py

@@ -14,6 +14,9 @@ games/catalogs/ and never in provenance/.
 
 No-Intro comes from the auto-datfile-generator release asset (Dat-o-Matic
 bans automated clients), Redump from redump.info's per-system datfile route.
+The mirror is dantob's fork: hugo19941994's original has failed its daily
+rebuild since Dat-o-Matic's 2026-07 re-layout (no-intro.zip last rebuilt
+2026-07-08), and the fork carries the fix (d4a9d769, 2026-07-31).
 Every download is cached under .cache/game-dats/ with what the server said
 about it, so a second run reads nothing from the network and writes the same
 bytes. A DAT's region tag is a market label, never the value a core computes.
@@ -49,8 +52,11 @@ SOURCES = ("no-intro", "redump")
 CACHE_DIR = Path(".cache") / "game-dats"
 OUTPUT_DIR = Path("games") / "catalogs"
 
+NOINTRO_MIRROR = "https://github.com/dantob/auto-datfile-generator"
+NOINTRO_RELEASE = "Daily_Rebuild"
 NOINTRO_RELEASE_API = (
-    "https://api.github.com/repos/hugo19941994/auto-datfile-generator/releases/tags/Daily_Rebuild"
+    NOINTRO_MIRROR.replace("https://github.com/", "https://api.github.com/repos/")
+    + f"/releases/tags/{NOINTRO_RELEASE}"
 )
 NOINTRO_ASSET = "no-intro.zip"
 REDUMP_DATFILE_URL = "https://redump.info/datfile/{code}"
@@ -120,7 +126,7 @@ LICENCES: dict[str, dict] = {
             "website or its operators is required for such use."
         ),
         "mirror": {
-            "repo": "https://github.com/hugo19941994/auto-datfile-generator",
+            "repo": NOINTRO_MIRROR,
             "licence": "MIT",
             "scope": "the mirror's scripts only, the DATs it republishes stay under the terms above",
         },

scripts/scraper/targets/emudeck_targets_scraper.py

@@ -16,6 +16,7 @@ import sys
 import urllib.error
 import urllib.request
 from datetime import datetime, timezone
+from pathlib import Path
 
 import yaml
 
@@ -32,6 +33,7 @@ STEAMOS_API = (
 WINDOWS_API = (
     "https://api.github.com/repos/EmuDeck/emudeck-we/contents/functions/EmuScripts"
 )
+RETROARCH_TARGETS = Path(__file__).resolve().parents[3] / "platforms" / "targets" / "retroarch.yml"
 
 def _fetch(url: str) -> str:
     headers = {"User-Agent": "retrobios-scraper/1.0"}
@@ -75,7 +77,7 @@ class Scraper(BaseTargetScraper):
         return self._revision
 
     def _fetch_cores_for_target(
-        self, api_url: str, label: str, arch: str = "x86_64"
+        self, api_url: str, label: str, retroarch_target: str
     ) -> list[str]:
         print(f"  fetching {label} EmuScripts...", file=sys.stderr)
         scripts = _list_emuscripts(api_url)
@@ -94,7 +96,7 @@ class Scraper(BaseTargetScraper):
         standalone_count = len(cores)
         # EmuDeck ships RetroArch = all its libretro cores are available
         if has_retroarch:
-            ra_cores = self._load_retroarch_cores(arch)
+            ra_cores = self._load_retroarch_cores(retroarch_target)
             for c in ra_cores:
                 if c not in seen:
                     seen.add(c)
@@ -108,24 +110,21 @@ class Scraper(BaseTargetScraper):
         return sorted(cores)
 
     @staticmethod
-    def _load_retroarch_cores(arch: str) -> list[str]:
-        """Load RetroArch target cores for given architecture."""
-        import os
-
-        target_path = os.path.join("platforms", "targets", "retroarch.yml")
-        with open(target_path) as f:
+    def _load_retroarch_cores(target: str, path: Path = RETROARCH_TARGETS) -> list[str]:
+        """The buildbot's libretro cores for one RetroArch target, by name:
+        linux-x86_64 and windows-x86_64 share an architecture, not a list."""
+        with path.open(encoding="utf-8") as f:
             data = yaml_load(f) or {}
-        # Find a target matching the architecture
-        for tinfo in data.get("targets", {}).values():
-            if tinfo.get("architecture") == arch and tinfo.get("cores"):
-                return tinfo["cores"]
-        raise RuntimeError(f"no {arch} core list in {target_path}")
+        cores = ((data.get("targets") or {}).get(target) or {}).get("cores")
+        if not cores:
+            raise RuntimeError(f"no {target} core list in {path}")
+        return cores
 
     def fetch_targets(self) -> dict:
         steamos_cores = self._fetch_cores_for_target(
-            f"{STEAMOS_API}?ref={self.revision}", "SteamOS"
+            f"{STEAMOS_API}?ref={self.revision}", "SteamOS", "linux-x86_64"
         )
-        windows_cores = self._fetch_cores_for_target(WINDOWS_API, "Windows")
+        windows_cores = self._fetch_cores_for_target(WINDOWS_API, "Windows", "windows-x86_64")
 
         targets: dict[str, dict] = {}
         if steamos_cores: