chore: merge tooling batch

scripts/check_freshness.py

@@ -40,6 +40,7 @@ import shutil
 import subprocess
 import sys
 import tempfile
+import urllib.request
 from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import asdict, dataclass, field
@@ -56,12 +57,14 @@ from common import (  # noqa: E402
     apply_target_overrides,
     list_registered_platforms,
     load_emulator_profiles,
+    load_platform_registry,
     upstream_profile_index,
     write_text_atomic,
     yaml_load,
 )
 from exporter import discover_exporters  # noqa: E402
 from exporter.baseline import build_native_model  # noqa: E402
+from scripts.scraper import game_dat_importer  # noqa: E402
 from scripts.scraper.redump_dat_scraper import fetch_snapshot  # noqa: E402
 from scripts.scraper.targets import discover_target_scrapers  # noqa: E402
 
@@ -80,7 +83,9 @@ INSTALLERS = ("install.sh", "install.ps1")
 
 SHOWN_ITEMS = 4  # items named in a diff summary before ", ..."
 SHOWN_NAMES = 12  # core names named in an unresolved row before ", ..."
-AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "ci", "profiles")
+AREAS = (
+    "platforms", "native", "targets", "romfolders", "coreinfo", "data", "catalogs", "ci", "profiles",
+)
 OK, STALE, UNKNOWN, ERROR, SKIPPED = "OK", "STALE", "UNKNOWN", "ERROR", "SKIPPED"
 # Every status a finding may carry, in the order the summary counts them.
 STATUSES = (STALE, UNKNOWN, ERROR, SKIPPED, OK)
@@ -723,6 +728,119 @@ def _age_days(stamp: str, now: datetime) -> int | None:
     return (now - then).days
 
 
+# --- rom folders ------------------------------------------------------------
+
+
+def romfolder_sources(platforms_dir: Path) -> dict[str, dict]:
+    """setup -> its platforms/romfolders/_sources/<setup>.yml."""
+    out = {}
+    for path in sorted((platforms_dir / "romfolders" / "_sources").glob("*.yml")):
+        with path.open(encoding="utf-8") as fh:
+            out[path.stem] = yaml_load(fh) or {}
+    return out
+
+
+def _os_source(doc: dict) -> dict:
+    """The source the setup is pinned on: `os`, else the first declared.
+    The others follow from it (an EmulationStation pinned by a package.mk)."""
+    sources = doc.get("sources") or {}
+    return sources.get("os") or next(iter(sources.values()), {})
+
+
+def platform_revision(setup: str, doc: dict, platforms_dir: Path) -> str:
+    """The revision the setup's own platform file was scraped at, when the
+    setup is a registered platform read from the same repository; '' else."""
+    path = platforms_dir / f"{setup}.yml"
+    if setup not in load_platform_registry(str(platforms_dir)) or not path.is_file():
+        return ""
+    with path.open(encoding="utf-8") as fh:
+        source = str((yaml_load(fh) or {}).get("source") or "")
+    repo = upstream.parse_repo(str(_os_source(doc).get("repo") or ""))
+    if repo is None or f"/{repo.slug.lower()}/" not in source.lower() + "/":
+        return ""
+    return export_native.source_revision(source)
+
+
+def romfolder_pin_drift(setup: str, doc: dict, platforms_dir: Path) -> str | None:
+    """Why the folders and the BIOS of one platform come from two revisions,
+    None when they do not or when only the forge can tell (a tag on one
+    side, a commit on the other)."""
+    revision = platform_revision(setup, doc, platforms_dir)
+    os_source = _os_source(doc)
+    tag, commit = os_source.get("tag"), str(os_source.get("commit") or "")
+    if not revision or revision in (tag, commit):
+        return None
+    if tag or len(revision) == len(commit):
+        return f"platforms/{setup}.yml was scraped at {revision[:12]}, the rom folders at {tag or commit[:12]}"
+    return None
+
+
+def check_romfolders(platforms_dir: Path, cache_dir: str, offline: bool) -> list[Finding]:
+    """The pins of platforms/romfolders/_sources/ against what they follow.
+
+    A setup that is a registered platform is read at the revision its
+    platform file was scraped at: the games-on-card mode otherwise installs
+    the BIOS of one release for the folders of another. Any other setup is
+    compared with its project's newest release, or with its branch head
+    when the project publishes none. A pin is moved by hand, then
+    `python -m scripts.scraper.romfolders --setup NAME` reads it again.
+    """
+    findings: list[Finding] = []
+    for setup, doc in romfolder_sources(platforms_dir).items():
+        os_source = _os_source(doc)
+        pin = str(os_source.get("tag") or str(os_source.get("commit") or "")[:12])
+        drift = romfolder_pin_drift(setup, doc, platforms_dir)
+        if drift:
+            findings.append(Finding("romfolders", setup, STALE, local=pin, detail=drift))
+            continue
+        revision = platform_revision(setup, doc, platforms_dir)
+        repo = upstream.parse_repo(str(os_source.get("repo") or ""))
+        commit = str(os_source.get("commit") or "")
+        if revision and revision in (os_source.get("tag"), commit):
+            findings.append(Finding("romfolders", setup, OK, local=pin,
+                                    detail=f"read where platforms/{setup}.yml is"))
+            continue
+        if offline:
+            findings.append(Finding("romfolders", setup, SKIPPED, local=pin, detail="offline"))
+            continue
+        if repo is None or not commit:
+            findings.append(Finding("romfolders", setup, UNKNOWN, local=pin, detail="no forge to ask"))
+            continue
+        findings.append(_romfolder_upstream(setup, repo, os_source, revision, cache_dir))
+    return findings
+
+
+def _romfolder_upstream(
+    setup: str, repo: upstream.Repo, os_source: dict, revision: str, cache_dir: str,
+) -> Finding:
+    commit, tag = str(os_source["commit"]), os_source.get("tag")
+    pin = str(tag or commit[:12])
+    if revision:
+        # The platform names a tag and the setup a commit: the tag's commit decides.
+        tagged = upstream.resolve_tag_commit(repo, revision, cache_dir)
+        if tagged is None:
+            return Finding("romfolders", setup, UNKNOWN, local=pin, detail=f"tag {revision} not listed")
+        if tagged == commit:
+            return Finding("romfolders", setup, OK, local=pin, detail=f"commit of {revision}")
+        return Finding("romfolders", setup, STALE, local=pin, remote=revision,
+                       detail=f"platforms/{setup}.yml was scraped at {revision} ({tagged[:12]})")
+    pinned_at = (upstream.commit_date(repo, commit, cache_dir) or "")[:10]
+    release = upstream.latest_release(repo, cache_dir)
+    if release is not None and release.date:
+        newer = release.tag != tag and release.date > pinned_at
+        return Finding("romfolders", setup, STALE if newer else OK, local=f"{pin} ({pinned_at})",
+                       remote=f"{release.tag} ({release.date})",
+                       detail="a release is newer than the pin" if newer else "newest release")
+    head = upstream.resolve_head(repo, cache_dir)
+    if head is None:
+        return Finding("romfolders", setup, UNKNOWN, local=pin, detail="branch head unavailable")
+    if head == commit:
+        return Finding("romfolders", setup, OK, local=pin, detail="branch head")
+    counts = upstream.ancestry(repo, commit, head, cache_dir)
+    behind = f"{counts[0]} commits behind the branch head" if counts else "the branch head moved"
+    return Finding("romfolders", setup, STALE, local=pin, remote=head[:12], detail=behind)
+
+
 # --- core-info --------------------------------------------------------------
 
 
@@ -822,6 +940,9 @@ def check_coreinfo(
     record_path: Path = CORE_INFO_RECORD,
 ) -> list[Finding]:
     blobs = core_info_blobs(cache_dir, offline)
+    if blobs is None and offline:
+        # Offline with no cached listing, nothing was asked of the upstream.
+        return [Finding("coreinfo", "libretro-core-info", SKIPPED, detail="offline")]
     if blobs is None:
         return [Finding("coreinfo", "libretro-core-info", UNKNOWN, detail="listing unavailable")]
     names = list(blobs)
@@ -931,9 +1052,87 @@ def check_catalogs(cache_dir: str, offline: bool) -> list[Finding]:
     findings.append(_check_tosec(offline))
     findings.append(_check_mame(cache_dir, offline))
     findings.append(_check_fbneo(cache_dir, offline))
+    findings.append(_check_nointro_games(cache_dir, offline))
+    findings.append(_check_redump_games(offline))
     return findings
 
 
+def _game_catalogs(source: str) -> list[dict]:
+    """The committed game catalogs of one source, games/catalogs/<source>/."""
+    return [_snapshot(path) for path in sorted((REPO_ROOT / "games" / "catalogs" / source).glob("*.json"))]
+
+
+def _no_intro_asset_date(cache_dir: str, offline: bool) -> str:
+    """The day the mirror last rebuilt its No-Intro pack, empty if unlisted."""
+    repo = upstream.parse_repo(NO_INTRO_MIRROR)
+    payload = upstream._api(  # noqa: SLF001
+        f"{repo.api_base}/repos/{repo.slug}/releases/tags/{NO_INTRO_RELEASE}",
+        cache_dir,
+        offline,
+        upstream.MOVING_TTL,
+    )
+    for asset in (payload.get("assets") if isinstance(payload, dict) else None) or []:
+        if isinstance(asset, dict) and asset.get("name") == NO_INTRO_ASSET:
+            return str(asset.get("updated_at", ""))[:10]
+    return ""
+
+
+def _check_nointro_games(cache_dir: str, offline: bool) -> Finding:
+    """games/catalogs/no-intro/ against the pack it was read from. The pack
+    holds every DAT: rebuilt after the catalogs were read, it may carry
+    games the games-on-card mode does not recognise."""
+    subject = "no-intro games"
+    fetched = min((str(doc.get("fetched", "")) for doc in _game_catalogs("no-intro")), default="")
+    if offline:
+        return Finding("catalogs", subject, SKIPPED, local=fetched, detail="offline")
+    updated = _no_intro_asset_date(cache_dir, offline)
+    if not updated:
+        return Finding("catalogs", subject, UNKNOWN, local=fetched, detail="asset not listed")
+    return Finding(
+        "catalogs", subject, STALE if updated > fetched else OK,
+        local=fetched, remote=updated, detail=f"{NO_INTRO_ASSET} rebuilt {updated}",
+    )
+
+
+def redump_datfile_version(code: str) -> str:
+    """The version redump.info serves for one datfile, read from the name it
+    gives the download: a HEAD request, nothing downloaded."""
+    url = game_dat_importer.REDUMP_DATFILE_URL.format(code=code)
+    request = urllib.request.Request(url, method="HEAD", headers={"User-Agent": "retrobios/1.0"})
+    with urllib.request.urlopen(request, timeout=60) as response:
+        disposition = response.headers.get("Content-Disposition", "")
+    named = game_dat_importer._DISPOSITION.search(disposition)  # noqa: SLF001
+    match = game_dat_importer._REDUMP_FILE.match(named["file"]) if named else None  # noqa: SLF001
+    if not match:
+        raise ValueError(f"{url} names no datfile")
+    return f"{match['date']} {match['time']}"
+
+
+def _check_redump_games(offline: bool) -> Finding:
+    """games/catalogs/redump/ against the datfiles redump.info serves now.
+    A disc dumped since is not recognised by the games-on-card mode."""
+    subject = "redump games"
+    local = {doc.get("system"): str(doc.get("version", "")) for doc in _game_catalogs("redump")}
+    newest = max(local.values(), default="")
+    if offline:
+        return Finding("catalogs", subject, SKIPPED, local=newest, detail="offline")
+    changed = []
+    remote_newest = ""
+    for code, (_name, system, _floor) in sorted(game_dat_importer.REDUMP_DATS.items()):
+        try:
+            remote = redump_datfile_version(code)
+        except (OSError, ValueError) as exc:
+            return Finding("catalogs", subject, UNKNOWN, local=newest, detail=str(exc))
+        remote_newest = max(remote_newest, remote)
+        if local.get(system) != remote:
+            changed.append(system)
+    return Finding(
+        "catalogs", subject, STALE if changed else OK, local=newest, remote=remote_newest,
+        detail=(f"changed: {', '.join(changed)}" if changed else f"{len(local)} DATs"),
+        items=changed,
+    )
+
+
 def _check_redump(offline: bool) -> Finding:
     local = _snapshot(REPO_ROOT / "provenance" / "redump.json")
     dats = local.get("dats") or {}
@@ -963,18 +1162,7 @@ def _check_no_intro(cache_dir: str, offline: bool) -> Finding:
     imported = str(local.get("imported_at", ""))
     if offline:
         return Finding("catalogs", "no-intro", SKIPPED, local=imported, detail="offline")
-    repo = upstream.parse_repo(NO_INTRO_MIRROR)
-    payload = upstream._api(  # noqa: SLF001
-        f"{repo.api_base}/repos/{repo.slug}/releases/tags/{NO_INTRO_RELEASE}",
-        cache_dir,
-        offline,
-        upstream.MOVING_TTL,
-    )
-    assets = payload.get("assets") if isinstance(payload, dict) else None
-    updated = ""
-    for asset in assets or []:
-        if isinstance(asset, dict) and asset.get("name") == NO_INTRO_ASSET:
-            updated = str(asset.get("updated_at", ""))[:10]
+    updated = _no_intro_asset_date(cache_dir, offline)
     if not updated:
         return Finding("catalogs", "no-intro", UNKNOWN, local=imported, detail="asset not listed")
     return Finding(
@@ -1410,8 +1598,10 @@ def render(findings: list[Finding]) -> str:
             cols = f"  {f.status:7} {f.subject:{width}}"
             if f.remote:
                 cols += f"  {f.local:{local_width}} -> {f.remote}"
-            elif f.local:
-                # An arrow pointing at nothing read as a lost value.
+            elif f.local or (f.detail and local_width):
+                # An arrow pointing at nothing read as a lost value. An empty
+                # local value still takes its column, or the detail starts
+                # where the other rows show theirs.
                 cols += f"  {f.local:{local_width}}"
             if f.detail:
                 cols += f"  {f.detail}"
@@ -1506,6 +1696,8 @@ def main() -> int:
             )
         if "targets" in areas:
             findings.extend(check_targets(platforms_dir, workdir, profiles, args.jobs, args.offline))
+    if "romfolders" in areas:
+        findings.extend(check_romfolders(platforms_dir, args.cache_dir, args.offline))
     if "coreinfo" in areas:
         findings.extend(check_coreinfo(profiles, args.cache_dir, args.offline))
     if "data" in areas:

scripts/cross_reference.py

@@ -19,6 +19,7 @@ import os
 import sys
 import zipfile
 import zlib
+from collections import Counter
 from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
@@ -676,6 +677,35 @@ def slot_key(f: dict, standalone: bool = False) -> tuple:
     )
 
 
+# Why an entry is left out of the comparison, in the order they are printed.
+SET_ASIDE = {
+    "other_versions": "other versions of a slot",
+    "not_read": "not read on this platform",
+    "outside_scope": "outside the pack",
+}
+
+
+def profile_breakdown(data: dict) -> str:
+    """Where each entry of a profile went, adding up to its file count.
+
+    The heading said 261 files, 7 declared and 238 undeclared, and the 16
+    others were nowhere: the versions a slot accepts, the entries the
+    platform never opens and the ones no pack carries.
+    """
+    # A report built without the per-entry tally (a fixture, a saved --json
+    # of an earlier run) falls back to its row counts.
+    parts = [f"{data.get('declared_files', data['platform_covered'])} declared by platforms",
+             f"{data.get('undeclared_files', data['gaps'])} undeclared"]
+    unsourceable = data.get("unsourceable_files", len(data.get("unsourceable") or []))
+    if unsourceable:
+        parts.append(f"{unsourceable} unsourceable")
+    aside = data.get("set_aside") or {}
+    detail = [f"{aside[key]} {label}" for key, label in SET_ASIDE.items() if aside.get(key)]
+    if detail:
+        parts.append(f"{sum(aside.values())} set aside ({', '.join(detail)})")
+    return ", ".join(parts)
+
+
 def _cross_reference_profile(
     emu_name: str,
     profile: dict,
@@ -735,6 +765,19 @@ def _cross_reference_profile(
     for f, is_standalone in layout_files:
         versions.setdefault(slot_key(f, is_standalone), []).append(f)
 
+    # What became of each entry, so the heading adds up: a file a platform
+    # declares, one it does not, or one set aside and why. An entry read by
+    # two builds keeps the first answer that compares it. Each layout lists
+    # the profile's entries in their order, so a position names one entry.
+    position = {id(f): i for view, _ in layouts for i, f in enumerate(view.get("files", []))}
+    fates: dict[int, str] = {}
+
+    def fate(f: dict, what: str) -> None:
+        key = position[id(f)]
+        current = fates.get(key)
+        if current is None or (current in SET_ASIDE and what not in SET_ASIDE):
+            fates[key] = what
+
     for f, is_standalone in layout_files:
         fname = f.get("name", "")
         # The destination the running build reads, as find_undeclared_files
@@ -753,13 +796,21 @@ def _cross_reference_profile(
         if len(layouts) > 1:
             # One file read at one path by both builds counts once.
             seen_key = (effective_path.casefold(), seen_key)
-        if not fname or seen_key in seen_files or unread_on(f, platform):
+        if not fname:
+            fate(f, "outside_scope")
+            continue
+        if seen_key in seen_files:
+            fate(f, "other_versions")
+            continue
+        if unread_on(f, platform):
+            fate(f, "not_read")
             continue
 
         # Collect unsourceable files separately (documented, not a gap)
         unsourceable_reason = f.get("unsourceable", "")
         if unsourceable_reason:
             seen_files.add(seen_key)
+            fate(f, "unsourceable")
             unsourceable_list.append({
                 "name": fname,
                 "required": f.get("required", False),
@@ -769,6 +820,7 @@ def _cross_reference_profile(
             continue
 
         if outside_gap_scope(f, is_standalone):
+            fate(f, "outside_scope")
             continue
 
         archive = f.get("archive")
@@ -784,6 +836,7 @@ def _cross_reference_profile(
 
         if in_platform:
             seen_files.add(seen_key)
+            fate(f, "declared")
             covered.append({
                 "name": fname,
                 "path": effective_path,
@@ -793,6 +846,7 @@ def _cross_reference_profile(
             continue
 
         seen_files.add(seen_key)
+        fate(f, "undeclared")
 
         if archive:
             archive_key = (
@@ -864,6 +918,7 @@ def _cross_reference_profile(
     for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]):
         gaps.append(ag)
 
+    tally = Counter(fates.values())
     report[emu_name] = {
         "emulator": profile.get("emulator", emu_name),
         "systems": systems,
@@ -877,6 +932,11 @@ def _cross_reference_profile(
         "gap_large_file": sum(1 for g in gaps if g["source"] == "large_file"),
         "gap_details": gaps,
         "unsourceable": unsourceable_list,
+        # Entries, not rows: an archive's members are one gap row.
+        "declared_files": tally["declared"],
+        "undeclared_files": tally["undeclared"],
+        "unsourceable_files": tally["unsourceable"],
+        "set_aside": {key: tally[key] for key in SET_ASIDE},
     }
 
 
@@ -977,11 +1037,7 @@ def print_report(
 
         systems = ", ".join(data["systems"])
         print(f"\n{data['emulator']}" + (f" ({systems})" if systems else ""))
-        print(
-            f"  {data['total_files']} files in profile, "
-            f"{data['platform_covered']} declared by platforms, "
-            f"{gaps} undeclared"
-        )
+        print(f"  {data['total_files']} files in profile: {profile_breakdown(data)}")
         print(f"  Gaps: {status}")
 
         for g in data["gap_details"]:

scripts/exporter/candidates.py

@@ -55,11 +55,16 @@ def _same(declared: str, accepted: str) -> bool:
     )
 
 
-def _accepts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
-    """True when the profile entry, or a member of its archive, accepts a
-    hash the platform entry declares."""
-    accepted = _entry_hashes(profile_entry)
-    return any(_same(d, a) for values in declared.values() for d in values for a in accepted)
+def _accepts(accepted: set[str], declared: dict[str, set[str]]) -> bool:
+    """True when the hashes a profile entry accepts, the members of its
+    archive included, hold one the platform entry declares."""
+    for values in declared.values():
+        if values & accepted:
+            return True
+        cut = [d for d in values if _CUT_MD5 <= len(d) < _FULL_MD5]
+        if cut and any(a.startswith(d) for d in cut for a in accepted):
+            return True
+    return False
 
 
 def _contradicts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
@@ -366,6 +371,10 @@ class EntryCategories:
         self._guessed = not config.get("cores")
         self._romsets = sorted(key for key in examined if _loads_romsets(profiles[key]))
         self._profiles = profiles
+        # The hashes each profile entry accepts, read once: every platform
+        # entry with a proof walks its reader's whole profile, and a MAME
+        # profile's archive members were hashed again for each.
+        self._accepted: dict[int, set[str]] = {}
         # One index per core and per kind, so a proof names its reader.
         self._readers: list[tuple[str, str, ReadEvidence]] = []
         for key in sorted(examined):
@@ -403,6 +412,12 @@ class EntryCategories:
                     sorted(unresolved_platform_cores(own, profiles)),
                 )
 
+    def _accepted_by(self, profile_entry: dict) -> set[str]:
+        accepted = self._accepted.get(id(profile_entry))
+        if accepted is None:
+            accepted = self._accepted[id(profile_entry)] = _entry_hashes(profile_entry)
+        return accepted
+
     def _matching(self, key: str, kind: str, entry: dict) -> list[dict]:
         """The profile entries of `key` that read the platform entry.
 
@@ -425,7 +440,7 @@ class EntryCategories:
             if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA):
                 continue
             cited = {k: f[k] for k in ("name", "source_ref") if f.get(k)}
-            if declared and _accepts(f, declared):
+            if declared and _accepts(self._accepted_by(f), declared):
                 by_hash.append(cited)
                 continue
             own = {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n}

scripts/generate_games_index.py

@@ -34,6 +34,7 @@ import argparse
 import gzip
 import hashlib
 import json
+import posixpath
 import sys
 from collections.abc import Iterable
 from dataclasses import dataclass
@@ -178,12 +179,25 @@ def _platform_required(platform: str, platforms_dir: str) -> dict[tuple[str, str
     return out
 
 
+def _narrowed(manifest: dict, cores: list[str], cache: dict) -> list[tuple[dict, str]]:
+    """The install list under some cores, each file with its lowercased
+    base name. The answer depends on the cores alone, and a platform asks
+    it once per system they run."""
+    key = (id(manifest), tuple(cores))
+    if key not in cache:
+        files, _omitted = install.narrow(
+            manifest["files"], manifest.get("omitted_files", []), [], cores, [])
+        cache[key] = [(entry, posixpath.basename(entry["dest"]).lower()) for entry in files]
+    return cache[key]
+
+
 def fallback_set(
     manifest: dict,
     platform_required: dict[tuple[str, str], bool],
     profiles: Profiles,
     system: str,
     profile: str | None,
+    narrowed: dict | None = None,
 ) -> dict:
     """The files an unrecognised game of one system keeps on one platform.
 
@@ -195,12 +209,10 @@ def fallback_set(
     """
     wanted = profiles.resolver.equivalents(system)
     cores = sorted(install.designated_cores(manifest, [profile])) if profile else []
-    files, _omitted = install.narrow(manifest["files"], manifest.get("omitted_files", []), [], cores, [])
     rows: list[list] = []
-    for entry in files:
+    for entry, base in _narrowed(manifest, cores, {} if narrowed is None else narrowed):
         systems = set(entry.get("systems") or [])
         dest = entry["dest"]
-        base = Path(dest).name.lower()
         if entry.get("cores") is None:
             if not systems & wanted:
                 continue
@@ -292,6 +304,7 @@ def build_setups(
     manifests: dict[str, dict] = {}
     required: dict[str, dict] = {}
     fallbacks: dict[str, dict[str, dict]] = {}
+    narrowed: dict = {}
     out: dict[str, dict] = {}
     coverage: dict[str, dict] = {}
     for setup, doc in sorted(setups.items()):
@@ -330,7 +343,8 @@ def build_setups(
                 for system in resolution.systems:
                     key = f"{system}|{chosen or ''}"
                     if key not in table:
-                        table[key] = fallback_set(manifest, required[platform], profiles, system, chosen)
+                        table[key] = fallback_set(manifest, required[platform], profiles, system,
+                                                  chosen, narrowed)
                     keys.append(key)
                 return keys
 

scripts/generate_installer_doc.py

@@ -223,8 +223,7 @@ def line_menu() -> str:
     """The line menu of a RetroArch run, g then Enter pressed, as install.py
     prints it for the RetroArch list (install/retroarch.json): the very
     function line mode runs, its questions answered here. The page carried
-    a copy typed by hand, whose keys and counts drifted from the installer
-    (code_audit_22, owner's note on which-pack)."""
+    a copy typed by hand, whose keys and counts drifted from the installer."""
     import io  # noqa: PLC0415
     from unittest import mock  # noqa: PLC0415
 
@@ -290,7 +289,7 @@ class _ListHandler(http.server.SimpleHTTPRequestHandler):
 class _LoopbackServer(http.server.ThreadingHTTPServer):
     """The list server, named by its number: HTTPServer.server_bind looks
     127.0.0.1 up with socket.getfqdn, a DNS query wherever /etc/hosts does
-    not name it (code_audit_24 P3-12)."""
+    not name it."""
 
     def server_bind(self) -> None:
         socketserver.TCPServer.server_bind(self)

scripts/generate_readme.py

@@ -929,7 +929,8 @@ the most useful contribution.
 
 From a clone, the file goes to `bios/Manufacturer/Console/filename`, and
 `python scripts/validate_pr.py path/to/file` names that folder for a file still
-outside `bios/`. A file over the browser's 25 MB limit goes in that way.
+outside `bios/`. Given a folder, such as an old card's BIOS folder, it checks
+every file under it. A file over the browser's 25 MB limit goes in that way.
 
 ## Add a platform
 

scripts/generate_site.py

@@ -3325,7 +3325,7 @@ def with_setup_table(page: str, coverages: dict, registry: dict | None = None) -
 
 # A platform's file lists are split by the share of a page they render to,
 # in thousandths: their elements against validate_site's MAX_PAGE_ELEMENTS,
-# their bytes against the 600,000 of test_site_r2_gate, whichever is
+# their bytes against the 600,000 of test_site_built_gate, whichever is
 # larger. Measured in Markdown characters, the lists split anew whenever
 # their markup changed: rows written as tables moved 38 more systems to
 # pages of their own.
@@ -4053,9 +4053,9 @@ def first_screen_words() -> list[str]:
     installer = _installer()
     # The screen it opens first: the setup found, or the list when none
     # is. Install comes on the next one, Choose systems beside it and named
-    # above on the page (tui_audit_16 P3-10).
+    # above on the page.
     # Nothing found, the folder comes before Install: which-pack left it
-    # out (tui_audit_17 P3-9).
+    # out.
     return [f"It asks **{installer.TUI_FOUND_TITLE}** (or **{installer.TUI_CHOOSE_TITLE}** "
             f"and **{installer.TUI_FOLDER_TITLE}**), then offers **{installer.TUI_INSTALL_ROW}**."]
 
@@ -4064,7 +4064,7 @@ def check_pass_line() -> str:
     """The pass line of "How to check": the words `--check` prints only when
     no file is missing or wrong (install.py EVERYTHING_IN_PLACE), read from
     the installer. Typed here, it kept "All files up to date." after the
-    installer said "Everything is in place." (tui_audit_15 P3-10)."""
+    installer said "Everything is in place."."""
     return f"All good when it says `{_installer().EVERYTHING_IN_PLACE}`"
 
 # Under "From your computer": a device with no network installs from the ZIP
@@ -10153,7 +10153,7 @@ def _emulator_part_page(
                              for n, (stem, _e) in enumerate(parts, 1)], number - 1),
         f"# {emu_name} files, part {number} of {len(parts)}",
         "",
-        # The crumbs end on the emulator, read from parent: (round 5, 7.5).
+        # The crumbs end on the emulator, read from parent:.
         f"This part holds {_count(len(files), 'entry', 'entries')}, "
         f"{_code(first)} to {_code(last)}. The other parts: {others}.",
         "",

scripts/pack_probe.py

@@ -173,6 +173,14 @@ def manifest_entries(manifest: dict) -> dict[str, str]:
     return out
 
 
+def installer_summary(stdout: str, system_dir: Path, save_dir: Path) -> str:
+    """The end of the installer's output, as a record keeps it. The scratch
+    folders are written as the options that named them: a record read
+    elsewhere has no use for the path of the machine that ran the probe."""
+    text = stdout.replace(str(system_dir), "<dest>").replace(str(save_dir), "<save-dir>")
+    return " ".join(text.split())[-400:]
+
+
 def installer_layout(platform: str, pack: Path, identity: dict, manifest_path: Path,
                      root: Path, narrow: Sequence[str] = ()) -> Layout:
     """install.py --dest on a scratch root, from the pack, offline. The
@@ -197,7 +205,7 @@ def installer_layout(platform: str, pack: Path, identity: dict, manifest_path: P
                      "sha256": file_sha(manifest_path, "sha256"),
                      "generated": manifest.get("generated"), "pack_files": manifest.get("pack_files")},
         "save_dir_files": len(walk(save_dir)) if save_dir.is_dir() else 0,
-        "installer_summary": " ".join(proc.stdout.split())[-400:],
+        "installer_summary": installer_summary(proc.stdout, system_dir, save_dir),
     }
     return Layout("installer", system_dir, files, source)
 

scripts/packextras.py

@@ -257,11 +257,27 @@ class ExtraResolver:
         # verify's replay: never downloads, and counts the release asset an
         # online build fetches, as it does for a platform entry.
         self.pending_assets = pending_assets
+        # Each profile's entries by slot, built on first use: every extra
+        # scanned its whole profile, 33 million entries in one verify --all.
+        self._slots: dict[str, dict[tuple[str, str], list[dict]]] = {}
+
+    def alternatives(self, fe: dict) -> list[dict]:
+        """The versions the extra's own profile accepts at its slot, as
+        validation.extra_alternatives finds them."""
+        from validation import slot_index
+
+        key = fe.get("source_profile") or ""
+        index = self._slots.get(key)
+        if index is None:
+            profile = (self.profiles or {}).get(key, {})
+            index = self._slots[key] = slot_index(profile.get("files") or [])
+        name = fe.get("source_name") or fe.get("name", "")
+        return index.get((name, fe.get("source_path") or name), [])
 
     def resolve(self, fe: dict, dest: str) -> tuple[str | None, str, str]:
         """(local copy or None, resolver status, the core's disagreement)."""
         from packresolve import pending_release_asset, resolve_file, run_is_offline
-        from validation import core_size_refusal, extra_alternatives, validated_choice
+        from validation import core_size_refusal, validated_choice
 
         local_path, status = resolve_file(
             fe, self.db, self.bios_dir, self.zip_contents,
@@ -285,9 +301,7 @@ class ExtraResolver:
         )
         # The platform never reads the bytes, the core does: a copy it turns
         # away by its own size check is not its file, on any platform.
-        if core_size_refusal(
-            fe, local_path, extra_alternatives(fe, self.profiles)
-        ):
+        if core_size_refusal(fe, local_path, self.alternatives(fe)):
             return None, "not_found", ""
         return local_path, status, disagreement
 

scripts/pipeline.py

@@ -405,7 +405,8 @@ def check_consistency(
         elif released is not None and not (v_names & released):
             print(f"  {v_label}: {v_ok}/{v_total} (no release pack, served by the installer)")
         else:
-            print(f"  {v_label}: {v_ok}/{v_total} (no separate pack)")
+            print(f"  {v_label}: MISSING pack: verify judged {v_ok}/{v_total} "
+                  "but the build wrote no pack for it")
             all_ok = False
 
     status = "OK" if all_ok else "FAILED"
@@ -602,6 +603,26 @@ def _refuse_unapplied_flags(
         args.source = "full"
 
 
+def _refuse_offline_without_caches(
+    parser: argparse.ArgumentParser, args: argparse.Namespace, platforms_dir: str = "platforms",
+) -> None:
+    """An offline run that builds packs needs the data directories cached.
+
+    On a fresh clone none is: --offline skips the refresh, and the run went
+    through the database, verify and every pack before failing at the end
+    of the build. Only an empty cache is refused here: with some cached, the
+    packs decide which ones they need and the build names the rest.
+    """
+    if not args.offline or args.skip_packs:
+        return
+    registry = load_data_dir_registry(platforms_dir)
+    if registry and not any(os.path.isdir(e.get("local_cache") or "") for e in registry.values()):
+        parser.error(
+            "no data directory is cached and --offline skips their refresh: run "
+            "python scripts/refresh_data_dirs.py once, or drop --offline"
+        )
+
+
 def _truth_steps(args: argparse.Namespace, results: dict) -> bool:
     """Steps 2c to 2e: truth model, its diff, the native export."""
     all_ok = True
@@ -722,6 +743,7 @@ def main() -> None:
     )
     args = parser.parse_args()
     _refuse_unapplied_flags(parser, args)
+    _refuse_offline_without_caches(parser, args)
     if not args.offline:
         # Read once and put in the environment every step inherits: the
         # scrapers and refresh_data_dirs went anonymous unless the user had

scripts/refresh_data_dirs.py

@@ -27,7 +27,7 @@ import zipfile
 from collections.abc import Iterator
 from pathlib import Path
 from artifacts import file_lock as _file_lock, set_plain_mode
-from common import github_token, platform_arg, require_platform, yaml_load
+from common import github_token, platform_arg, require_platform, suggest_names, yaml_load
 
 try:
     import yaml
@@ -607,7 +607,7 @@ def main() -> None:
 
     if args.key:
         if args.key not in registry:
-            log.error("unknown key: %s (available: %s)", args.key, ", ".join(registry))
+            log.error("unknown key: %s. %s", args.key, suggest_names([args.key], list(registry), "--dry-run"))
             raise SystemExit(1)
         allowed = registry[args.key].get("for_platforms")
         if args.platform and allowed and args.platform not in allowed:

scripts/refresh_stale.py

@@ -416,6 +416,18 @@ def _command_for(
                 "fbneo",
                 "--fetch",
             ]
+        if subject in ("no-intro games", "redump games"):
+            # The game DATs download without a browser, unlike the BIOS ones.
+            return [
+                sys.executable,
+                "-m",
+                "scripts.scraper.game_dat_importer",
+                "--source",
+                subject.split()[0],
+                "--fetch",
+                "--refresh",
+                "--floor",
+            ]
         # redump / no-intro / tosec need manual DAT download.
         return None
 

scripts/restore_large_files.py

@@ -40,6 +40,11 @@ def index_cache(cache_dir: str) -> dict[str, str]:
     """Map SHA1 to cached file path for every asset in the cache."""
     index: dict[str, str] = {}
     for name in sorted(os.listdir(cache_dir)):
+        # A download in progress writes `<asset>.<random>.tmp` beside the
+        # cache and renames it when done: it is partial, and may be gone by
+        # the time it is opened.
+        if name.endswith(".tmp"):
+            continue
         path = os.path.join(cache_dir, name)
         if not os.path.isfile(path):
             continue

scripts/scraper/targets/batocera_targets_scraper.py

@@ -29,27 +29,15 @@ from . import BaseTargetScraper
 
 PLATFORM_NAME = "batocera"
 
-GITHUB_API = "https://api.github.com/repos/batocera-linux/batocera.linux/contents"
 try:
     from ..base_scraper import github_headers
 except ImportError:
     from scraper.base_scraper import github_headers
 
-RAW_BASE = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/{tag}"
-
-CONFIG_IN_PATH = "package/batocera/core/batocera-system/Config.in"
-ES_SYSTEMS_PATH = "package/batocera/emulationstation/batocera-es-system/es_systems.yml"
-
 _TARGET_FLAG_RE = re.compile(r"^(BR2_PACKAGE_BATOCERA_TARGET_\w+)=y", re.MULTILINE)
 
-# Meta-flag definition: "if COND\n\tconfig DERIVED_FLAG\n\t...\nendif"
-_META_BLOCK_RE = re.compile(
-    r"^if\s+((?:[^\n]|\\\n)+?)\n"  # condition (may span lines via \)
-    r"(?:.*?\n)*?"  # optional lines before the config
-    r"\s+config\s+(BR2_PACKAGE_\w+)"  # derived flag name
-    r".*?^endif",  # end of block
-    re.MULTILINE | re.DOTALL,
-)
+_IF_RE = re.compile(r"^if\s+(.*)$")
+_DEFAULT_Y_RE = re.compile(r"^\s*default\s+y\s*(?:#.*)?$")
 
 
 def _fetch(url: str, headers: dict | None = None) -> str:
@@ -156,13 +144,30 @@ def _parse_meta_flags(text: str) -> list[tuple[str, str]]:
     """Return [(derived_flag, condition_str)] from top-level if/endif blocks.
 
     These define derived flags like BR2_PACKAGE_BATOCERA_TARGET_X86_64_ANY,
-    BR2_PACKAGE_BATOCERA_GLES3, etc.
+    BR2_PACKAGE_BATOCERA_GLES3, etc.: a config inside `if COND` that
+    defaults to y is on exactly when COND holds. Read line by line: a
+    condition continued with a backslash keeps its later lines, and a block
+    whose config is not a package (KNULLI's BR2_KERNEL_LEGACY_3_10) cannot
+    reach into the next one.
     """
     results: list[tuple[str, str]] = []
-    for m in _META_BLOCK_RE.finditer(text):
-        cond = _normalise_condition(m.group(1))
-        flag = m.group(2)
-        results.append((flag, cond))
+    condition: str | None = None
+    config: str | None = None
+    for line in re.sub(r"\\\n", " ", text).splitlines():
+        start = _IF_RE.match(line)
+        if start:
+            condition, config = _normalise_condition(start.group(1)), None
+            continue
+        if condition is None:
+            continue
+        if line.startswith("endif"):
+            condition = config = None
+            continue
+        found = _CONFIG_LINE_RE.match(line)
+        if found:
+            config = found.group(1)
+        elif config and _DEFAULT_Y_RE.match(line):
+            results.append((config, condition))
     return results
 
 
@@ -174,8 +179,6 @@ BUILDROOT_KCONFIG = (
     "package/mesa3d/Config.in",
     "package/libglvnd/Config.in",
 )
-BUILDROOT_RAW = "https://raw.githubusercontent.com/batocera-linux/buildroot/{sha}"
-SYSTEM_SYMBOL = "BR2_PACKAGE_BATOCERA_SYSTEM"
 
 _CONFIG_LINE_RE = re.compile(r"^\s*(?:menu)?config\s+(BR2_\w+)")
 _OWNED_SELECT_RE = re.compile(r"^\s+select\s+(BR2_\w+)(?:\s+if\s+([^#]+))?")
@@ -217,22 +220,26 @@ def _active_symbols(
 
     Run to a fixed point, as Kconfig does: a select can turn on a config
     whose own selects then fire, and a derived flag can follow from either.
+    Each pass rebuilds the set from the board's own flags, so a select
+    guarded by a negation (`if !BR2_PACKAGE_BATOCERA_GLES3`) that fired
+    before GLES3 was derived is dropped once it is: accumulating kept
+    duckstation-legacy on every GLES3 board.
     """
-    active = set(board_flags)
-    while True:
-        before = len(active)
-        frozen = frozenset(active)
-        active |= {
+    base = frozenset(board_flags)
+    active = base
+    for _ in range(64):
+        following = base | {
             derived for derived, condition in meta_rules
-            if _condition_holds(condition, frozen)
-        }
-        active |= {
+            if _condition_holds(condition, active)
+        } | {
             selected
             for owner, selected, condition in rules
-            if owner in frozen and _condition_holds(condition, frozen)
+            if owner in active and _condition_holds(condition, active)
         }
-        if len(active) == before:
-            return frozenset(active)
+        if following == active:
+            return active
+        active = frozenset(following)
+    raise RuntimeError("Kconfig selects do not settle: a negated condition cycles")
 
 
 def _parse_es_systems(text: str) -> dict[str, list[str]]:
@@ -279,8 +286,25 @@ def _parse_es_systems(text: str) -> dict[str, list[str]]:
     return package_to_emulators
 
 
-def _arch_from_flag(flag: str) -> str:
-    """Guess architecture from board flag name."""
+# Buildroot's architecture choice, as a board sets it (arch/Config.in).
+_ARCH_SYMBOLS = {
+    "BR2_aarch64": "aarch64",
+    "BR2_arm": "arm",
+    "BR2_x86_64": "x86_64",
+    "BR2_i386": "x86",
+    "BR2_riscv": "riscv64",
+}
+
+
+def _arch_from_flag(flag: str, board_flags: set[str] | frozenset[str] = frozenset()) -> str:
+    """The board's Buildroot architecture, else a guess from its target flag.
+
+    KNULLI's R16, RK3128 and ATM7039 images are 32-bit ARM (BR2_arm=y):
+    the target flag alone does not say so.
+    """
+    for symbol, arch in _ARCH_SYMBOLS.items():
+        if symbol in board_flags:
+            return arch
     low = flag.lower()
     if "x86_64" in low or "zen3" in low:
         return "x86_64"
@@ -290,35 +314,51 @@ def _arch_from_flag(flag: str) -> str:
 
 
 class Scraper(BaseTargetScraper):
-    """Cross-references Batocera boards, Config.in, and es_systems to build target lists."""
+    """Cross-references Batocera boards, Config.in, and es_systems to build target lists.
 
-    def __init__(
-        self, url: str = "https://github.com/batocera-linux/batocera.linux"
-    ) -> None:
-        super().__init__(url=url)
+    A Buildroot tree built on Batocera's model (KNULLI) overrides the class
+    attributes: its repository, board prefix, system Config.in, es_systems
+    and the config every image turns on.
+    """
+
+    PLATFORM = PLATFORM_NAME
+    REPO = "batocera-linux/batocera.linux"
+    BOARD_PREFIX = "batocera-"
+    CONFIG_IN_PATH = "package/batocera/core/batocera-system/Config.in"
+    ES_SYSTEMS_PATH = "package/batocera/emulationstation/batocera-es-system/es_systems.yml"
+    SYSTEM_SYMBOL = "BR2_PACKAGE_BATOCERA_SYSTEM"
+    BUILDROOT_REPO = "batocera-linux/buildroot"
+
+    def __init__(self, url: str = "") -> None:
+        super().__init__(url=url or f"https://github.com/{self.REPO}")
         self.tag = ""
 
     def _raw(self, path: str) -> str:
-        return f"{RAW_BASE.format(tag=self.tag)}/{path}"
+        return f"https://raw.githubusercontent.com/{self.REPO}/{self.tag}/{path}"
+
+    def _api(self, path: str) -> str:
+        return f"https://api.github.com/repos/{self.REPO}/contents/{path}?ref={self.tag}"
 
     def _list_boards(self) -> list[str]:
-        """List batocera-*.board files from configs/ via GitHub API."""
-        data = _fetch_json(f"{GITHUB_API}/configs?ref={self.tag}")
+        """List <prefix>*.board files from configs/ via GitHub API."""
+        data = _fetch_json(self._api("configs"))
         boards = [
             item["name"]
             for item in data
             if isinstance(item, dict)
-            and item.get("name", "").startswith("batocera-")
+            and item.get("name", "").startswith(self.BOARD_PREFIX)
             and item.get("name", "").endswith(".board")
         ] if isinstance(data, list) else []
         if not boards:
-            raise RuntimeError(f"no batocera-*.board file in configs/ at {self.tag}")
+            raise RuntimeError(
+                f"no {self.BOARD_PREFIX}*.board file in configs/ at {self.tag}"
+            )
         return boards
 
     def _fetch_board(self, board_name: str) -> tuple[str | None, set[str]]:
         """A board's target flag and every flag it sets, includes followed.
 
-        batocera-board.common turns on BATOCERA_SYSTEM, the config whose
+        <prefix>board.common turns on the system symbol, the config whose
         selects bring in the display stack and every emulator group.
         """
         flags: set[str] = set()
@@ -343,24 +383,25 @@ class Scraper(BaseTargetScraper):
         return target, flags
 
     def _buildroot_rules(self) -> list[tuple[str, str, str]]:
-        """Buildroot's selects for the symbols Batocera tests, at its pin."""
-        pin = _fetch_json(f"{GITHUB_API}/buildroot?ref={self.tag}")
+        """Buildroot's selects for the symbols the tree tests, at its pin."""
+        pin = _fetch_json(self._api("buildroot"))
         sha = pin.get("sha") if isinstance(pin, dict) else None
         if not sha:
             raise RuntimeError(f"no buildroot submodule commit at {self.tag}")
+        raw = f"https://raw.githubusercontent.com/{self.BUILDROOT_REPO}/{sha}"
         rules: list[tuple[str, str, str]] = []
         for path in BUILDROOT_KCONFIG:
-            rules += _parse_owned_selects(_fetch(f"{BUILDROOT_RAW.format(sha=sha)}/{path}"))
+            rules += _parse_owned_selects(_fetch(f"{raw}/{path}"))
         return rules
 
     def fetch_targets(self) -> dict:
         """Build per-board emulator availability map."""
-        self.tag = self.pinned_revision(PLATFORM_NAME)
+        self.tag = self.pinned_revision(self.PLATFORM)
         print("  fetching board list...", file=sys.stderr)
         boards = self._list_boards()
 
         print("  fetching Config.in...", file=sys.stderr)
-        config_in_url = self._raw(CONFIG_IN_PATH)
+        config_in_url = self._raw(self.CONFIG_IN_PATH)
         config_in_text = _fetch(config_in_url)
         if not config_in_text:
             raise RuntimeError(f"empty Config.in from {config_in_url}")
@@ -376,7 +417,7 @@ class Scraper(BaseTargetScraper):
         rules += self._buildroot_rules()
 
         print("  fetching es_systems.yml...", file=sys.stderr)
-        es_systems_url = self._raw(ES_SYSTEMS_PATH)
+        es_systems_url = self._raw(self.ES_SYSTEMS_PATH)
         es_text = _fetch(es_systems_url)
         if not es_text:
             raise RuntimeError(f"empty es_systems.yml from {es_systems_url}")
@@ -390,7 +431,7 @@ class Scraper(BaseTargetScraper):
 
         targets: dict[str, dict] = {}
         for board_name in sorted(boards):
-            target_key = board_name.removeprefix("batocera-").removesuffix(".board")
+            target_key = board_name.removeprefix(self.BOARD_PREFIX).removesuffix(".board")
             print(f"  processing {target_key}...", file=sys.stderr)
             primary_flag, board_flags = self._fetch_board(board_name)
             if primary_flag is None:
@@ -400,8 +441,8 @@ class Scraper(BaseTargetScraper):
             selected_packages = _active_symbols(board_flags, meta_rules, rules)
             # x86_wow64 builds the 32-bit Wine package the x86_64 image
             # carries: no batocera-board.common, no system, no target.
-            if SYSTEM_SYMBOL not in selected_packages:
-                print(f"    {board_name} builds no Batocera image", file=sys.stderr)
+            if self.SYSTEM_SYMBOL not in selected_packages:
+                print(f"    {board_name} builds no system image", file=sys.stderr)
                 continue
 
             # Map selected packages to emulator names via es_systems.yml
@@ -410,7 +451,7 @@ class Scraper(BaseTargetScraper):
                 for emu in package_to_emulators.get(pkg, []):
                     emulators.add(emu)
 
-            arch = _arch_from_flag(primary_flag)
+            arch = _arch_from_flag(primary_flag, board_flags)
             targets[target_key] = {
                 "architecture": arch,
                 "cores": sorted(emulators),
@@ -421,23 +462,23 @@ class Scraper(BaseTargetScraper):
             )
 
         return {
-            "platform": "batocera",
+            "platform": self.PLATFORM,
             "source": f"{self.url}/tree/{self.tag}",
             "scraped_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
             "targets": targets,
         }
 
 
-def main() -> None:
+def main(scraper_class: type[Scraper] = Scraper) -> None:
     parser = argparse.ArgumentParser(
         allow_abbrev=False,
-        description="Scrape Batocera per-board emulator targets"
+        description=f"Scrape {scraper_class.PLATFORM} per-board emulator targets"
     )
     parser.add_argument("--dry-run", action="store_true", help="Show target summary")
     parser.add_argument("--output", "-o", help="Output YAML file")
     args = parser.parse_args()
 
-    scraper = Scraper()
+    scraper = scraper_class()
     data = scraper.fetch_targets()
 
     if args.dry_run:

scripts/scraper/targets/knulli_targets_scraper.py

@@ -0,0 +1,42 @@
+"""Scraper for KNULLI per-board emulator availability.
+
+KNULLI is a Buildroot external tree on Batocera's model, read with the
+Batocera scraper's Kconfig resolution. Sources (knulli-cfw/knulli-linux, at
+the tag platforms/knulli.yml names):
+  - configs/knulli-*.board -- board definitions, each sets
+    BR2_PACKAGE_BATOCERA_TARGET_* and includes knulli-board.common
+  - package/system/knulli-system/Config.in -- the system config, sourced by
+    the root Config.in in place of Batocera's batocera-system/Config.in
+  - package/emulationstation/knulli-es-system/es_systems.yml
+    -- emulator requireAnyOf flag mapping
+Sources (knulli-cfw/buildroot, at the submodule commit the tag pins):
+  - arch/Config.in, package/mesa3d/Config.in, package/libglvnd/Config.in
+
+The *_armhf_libs boards include knulli-board.libs, which builds 32-bit
+libraries for the matching image and turns no system on: they are no target.
+"""
+
+from __future__ import annotations
+
+from .batocera_targets_scraper import Scraper as BatoceraScraper
+from .batocera_targets_scraper import main as batocera_main
+
+PLATFORM_NAME = "knulli"
+
+
+class Scraper(BatoceraScraper):
+    PLATFORM = PLATFORM_NAME
+    REPO = "knulli-cfw/knulli-linux"
+    BOARD_PREFIX = "knulli-"
+    CONFIG_IN_PATH = "package/system/knulli-system/Config.in"
+    ES_SYSTEMS_PATH = "package/emulationstation/knulli-es-system/es_systems.yml"
+    SYSTEM_SYMBOL = "BR2_PACKAGE_KNULLI_SYSTEM"
+    BUILDROOT_REPO = "knulli-cfw/buildroot"
+
+
+def main() -> None:
+    batocera_main(Scraper)
+
+
+if __name__ == "__main__":
+    main()

scripts/sitebootstrap.py

@@ -45,7 +45,7 @@ PIN: dict[str, re.Pattern[str]] = {
 # What the served copy says of a refused installer. It names a commit's
 # install.py, which never changes, and both downloaders fail on a short
 # transfer before the hash is read: neither a cut download nor caches of
-# two ages explain a mismatch there (code_audit_21 P3-1).
+# two ages explain a mismatch there.
 SERVED_ADVICE = ("The file was changed on its way here, most often by a proxy or an "
                  "antivirus that rewrites downloads. Run the command again from another "
                  "network, or with that web protection paused.")
@@ -73,7 +73,7 @@ def served_text(name: str, root: Path, commit: str) -> str:
     advice fitted to that. A pin that is not the SHA-256 of root's
     install.py stops the build: published, that copy refused every install
     until the next good deploy, where a failed build keeps the last good
-    pair served (code_audit_21 P3-2)."""
+    pair served."""
     source, line = SOURCES[name]
     text = (root / source).read_text(encoding="utf-8")
     pinned = PIN[name].findall(text)

scripts/validate_pr.py

@@ -23,6 +23,7 @@ import os
 import re
 import subprocess
 import sys
+from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
 from common import (
@@ -97,10 +98,13 @@ class ValidationResult:
         lines = [f"### {status} `{self.filename}`"]
         lines.append("")
         lines.append(f"- **Path**: `{self.filepath}`")
-        lines.append(f"- **Size**: {self.size:,} bytes")
-        lines.append(f"- **SHA1**: `{self.sha1}`")
-        lines.append(f"- **MD5**: `{self.md5}`")
-        lines.append(f"- **CRC32**: `{self.crc32}`")
+        # A path refused before it was read has no size or digest to show:
+        # empty fields read as an empty file contributed.
+        if self.sha1:
+            lines.append(f"- **Size**: {self.size:,} bytes")
+            lines.append(f"- **SHA1**: `{self.sha1}`")
+            lines.append(f"- **MD5**: `{self.md5}`")
+            lines.append(f"- **CRC32**: `{self.crc32}`")
         lines.append("")
 
         for status_str, message in self.checks:
@@ -312,11 +316,17 @@ def validate_file(
         return result
 
     if not os.path.exists(filepath):
-        result.add_check(False, f"File not found: {filepath}")
+        result.add_check(False, f"File not found: {filepath}. A path is read from the folder "
+                                "the command runs in, or pass --changed to check the files "
+                                "changed under bios/")
+        return result
+
+    if os.path.isdir(filepath):
+        result.add_check(False, "Folder with no file in it")
         return result
 
     if not os.path.isfile(filepath):
-        result.add_check(False, "Not a regular file")
+        result.add_check(False, "Not a regular file: a device, a pipe or a socket")
         return result
 
     result.size = os.path.getsize(filepath)
@@ -623,6 +633,26 @@ def get_changed_files() -> list[str]:
     ]
 
 
+def expand_folders(paths: list[str]) -> list[str]:
+    """Each folder named replaced by the files under it, subfolders included.
+
+    A card's BIOS folder was refused as "Not a regular file", and the
+    contributor had to list its files by hand. A link inside the folder
+    stays listed, so it is refused like a named one, and an empty folder
+    stays as itself to be reported.
+    """
+    out: list[str] = []
+    for path in paths:
+        if not os.path.isdir(path) or os.path.islink(path):
+            out.append(path)
+            continue
+        found = sorted(
+            str(f) for f in Path(path).rglob("*") if f.is_symlink() or f.is_file()
+        )
+        out.extend(found or [path])
+    return out
+
+
 def _files_to_validate(
     args: argparse.Namespace, parser: argparse.ArgumentParser
 ) -> list:
@@ -634,7 +664,7 @@ def _files_to_validate(
     if not args.changed:
         if not args.files:
             parser.error("No files specified. Use --changed or provide file paths.")
-        return args.files
+        return expand_folders(args.files)
     try:
         files = get_changed_files()
     except (RuntimeError, OSError) as exc:
@@ -737,9 +767,10 @@ def main() -> None:
         for r in results:
             status = "PASS" if r.passed else "FAIL"
             print(f"\n[{status}] {r.filepath}")
-            print(f"  SHA1: {r.sha1}")
-            print(f"  MD5:  {r.md5}")
-            print(f"  Size: {r.size:,}")
+            if r.sha1:
+                print(f"  SHA1: {r.sha1}")
+                print(f"  MD5:  {r.md5}")
+                print(f"  Size: {r.size:,}")
             for s, m in r.checks:
                 marker = (
                     "ok"

scripts/validate_schemas.py

@@ -39,6 +39,22 @@ def _errors(validator: Draft202012Validator, data: object, label: str) -> list[s
     return out
 
 
+_SOURCES: dict[tuple[Path, int, int], object] = {}
+
+
+def _load_source(path: Path) -> object:
+    """A source YAML, parsed once per run whichever check reads it: the
+    schema pass, the data directory refs and the layout check each parsed
+    all 649 profiles again. Keyed by size and mtime too, so a file written
+    again in the same process is read again."""
+    stat = path.stat()
+    key = (path, stat.st_mtime_ns, stat.st_size)
+    if key not in _SOURCES:
+        with path.open(encoding="utf-8") as handle:
+            _SOURCES[key] = yaml_load(handle)
+    return _SOURCES[key]
+
+
 def _validate_yaml_directory(
     directory: Path,
     schema_name: str,
@@ -51,8 +67,7 @@ def _validate_yaml_directory(
         if skip_private and path.name.startswith("_"):
             continue
         try:
-            with path.open(encoding="utf-8") as handle:
-                data = yaml_load(handle)
+            data = _load_source(path)
         except (OSError, yaml.YAMLError) as exc:
             out.append(f"{path.relative_to(ROOT)}: {exc}")
             continue
@@ -273,8 +288,7 @@ def _unknown_data_dir_refs() -> list[str]:
         for path in source_yaml_paths(directory):
             if path.name.startswith("_"):
                 continue
-            with path.open(encoding="utf-8") as handle:
-                document = yaml_load(handle) or {}
+            document = _load_source(path) or {}
             systems = document.get("systems")
             holders = [document, *(systems.values() if isinstance(systems, dict) else [])]
             for holder in holders:
@@ -299,8 +313,7 @@ def _repository_layout_in_profiles() -> list[str]:
     """
     out: list[str] = []
     for path in source_yaml_paths(ROOT / "emulators"):
-        with path.open(encoding="utf-8") as handle:
-            document = yaml_load(handle) or {}
+        document = _load_source(path) or {}
         for entry in document.get("files") or []:
             if not isinstance(entry, dict):
                 continue
@@ -688,6 +701,34 @@ def _games_source_errors() -> list[str]:
     return out
 
 
+def _arcade_pin_errors() -> list[str]:
+    """games/arcade/<profile>.json read at the revision and version its
+    profile pins. A bump of the profile left the game-to-BIOS map of the old
+    code in place, and the installer kept serving it."""
+    out: list[str] = []
+    for path in sorted((ROOT / "games" / "arcade").glob("*.json")):
+        doc = _load_json(path)
+        profile_path = ROOT / "emulators" / f"{doc.get('profile')}.yml"
+        label = str(path.relative_to(ROOT))
+        if not profile_path.is_file():
+            out.append(f"{label}: no profile emulators/{profile_path.name}")
+            continue
+        profile = _load_source(profile_path) or {}
+        pinned = profile.get("source_commit")
+        pins = set(pinned.values()) if isinstance(pinned, dict) else {pinned}
+        commit = (doc.get("source") or {}).get("commit")
+        drifted = []
+        if commit and commit not in pins:
+            drifted.append(f"read at {commit[:12]}, the profile pins {pinned}")
+        if doc.get("core_version") != profile.get("core_version"):
+            drifted.append(f"core_version {doc.get('core_version')!r}, "
+                           f"the profile says {profile.get('core_version')!r}")
+        if drifted:
+            out.append(f"{label}: {'; '.join(drifted)} "
+                       f"(python -m scripts.scraper.arcade_parent_importer --profile {doc.get('profile')})")
+    return out
+
+
 def _games_index_errors() -> list[str]:
     """install/games/: each document against its kind, and the index
     against the files it lists (size and sha256)."""
@@ -752,6 +793,7 @@ def main() -> int:
     errors.extend(_measurement_errors())
     errors.extend(_romfolder_errors())
     errors.extend(_games_source_errors())
+    errors.extend(_arcade_pin_errors())
 
     if not args.source_only:
         database_path = ROOT / "database.json"

scripts/validation.py

@@ -1004,6 +1004,17 @@ def same_slot_entries(files: list, name: str, path: str) -> list[dict]:
     ]
 
 
+def slot_index(files: list) -> dict[tuple[str, str], list[dict]]:
+    """A profile's entries keyed by (name, path), as same_slot_entries
+    matches them: one pass, then one lookup per extra."""
+    index: dict[tuple[str, str], list[dict]] = {}
+    for entry in files or []:
+        if isinstance(entry, dict):
+            name = entry.get("name")
+            index.setdefault((name, entry.get("path") or name), []).append(entry)
+    return index
+
+
 def extra_alternatives(fe: dict, profiles: Mapping | None) -> list[dict]:
     """The versions the extra's own profile accepts at the extra's slot."""
     if not profiles: