chore: merge main into Sony pages

scripts/artifacts.py

@@ -7,6 +7,7 @@ it is only meaningful if the comparison ignores the hour."""
 from __future__ import annotations
 
 from collections.abc import Callable, Iterator
+from typing import BinaryIO
 import contextlib
 import json
 import os
@@ -159,6 +160,29 @@ def copy_file_atomic(source: str, path: str) -> None:
             os.unlink(scratch)
         raise
 
+
+def copy_stream_atomic(stream: BinaryIO, path: str) -> None:
+    """Stream a download into place whole or not at all.
+
+    A fixed scratch name let two runs interleave their bytes in one file of
+    the right size, which the rename then promoted."""
+    import shutil
+
+    directory = os.path.dirname(os.path.abspath(path))
+    handle, scratch = tempfile.mkstemp(
+        dir=directory, prefix=f".{os.path.basename(path)}.", suffix=".tmp"
+    )
+    try:
+        with os.fdopen(handle, "wb") as f:
+            shutil.copyfileobj(stream, f)
+        set_plain_mode(scratch)
+        os.replace(scratch, path)
+    except BaseException:
+        with contextlib.suppress(OSError):
+            os.unlink(scratch)
+        raise
+
+
 def _strip_timestamps(text: str) -> str:
     """Remove known timestamp patterns for content comparison."""
     result = text

scripts/common.py

@@ -2663,6 +2663,106 @@ SYSTEM_ALIASES: dict[str, str] = {
     # profiles use only through these.
     "sega-segacd": "sega-megacd",
     "nes-fds": "nintendo-fds",
+    # One machine, two spellings across platform folders and profiles:
+    # Recalbox's apple2gs, Batocera's astrocade, KNULLI and RetroBat's
+    # astrocde, MiSTer's creativision, Batocera's gamepocketcomputer
+    # folder, BizHawk's Super Game Boy, the FBA 2012 board names.
+    "apple2gs": "apple-iigs",
+    "astrocade": "bally-astrocade",
+    "astrocde": "bally-astrocade",
+    "creativision": "crvision",
+    "gamepocketcomputer": "gamepock",
+    "nintendo-super-game-boy": "nintendo-sgb",
+    "cps1": "capcom-cps1",
+    "cps2": "capcom-cps2",
+    "cps3": "capcom-cps3",
+    "pgm": "igs-pgm",
+    # Rom folder names the setups of platforms/romfolders/ give a machine,
+    # each proven by the source_ref of the folder row that names it: the
+    # short tags of Onion, CrossMix-OS, Koriki and the MinUI family, the
+    # number words of Koriki and plumOS, the region and hack folders of
+    # ES-DE, the MSU-1 folders of the ES family, the arcade cores named as
+    # folders.
+    "a2600": "atari-2600",
+    "atari2600": "atari-2600",
+    "fiftytwohundred": "atari-5200",
+    "seventyeighthundred": "atari-7800",
+    "atarixegs": "xegs",
+    "atarijaguarcd": "atari-jaguarcd",
+    "fc": "nintendo-nes",
+    "nesh": "nintendo-nes",
+    "sfc": "nintendo-snes",
+    "supa": "nintendo-snes",
+    "snesh": "nintendo-snes",
+    "snesna": "nintendo-snes",
+    "snesmsu1": "nintendo-snes",
+    "snes-msu1": "nintendo-snes",
+    "msu1": "nintendo-snes",
+    "vb": "nintendo-virtualboy",
+    "gc": "nintendo-gamecube",
+    "gbh": "nintendo-gb",
+    "gb2players": "nintendo-gb",
+    "gbch": "nintendo-gbc",
+    "gbc2players": "nintendo-gbc",
+    "gbah": "nintendo-gba",
+    "mgba": "nintendo-gba",
+    "poke": "nintendo-pokemon-mini",
+    "pkm": "nintendo-pokemon-mini",
+    "md": "sega-megadrive",
+    "genh": "sega-megadrive",
+    "megadriveh": "sega-megadrive",
+    "megadrivejp": "sega-megadrive",
+    "megadrive-japan": "sega-megadrive",
+    "msumd": "sega-megadrive",
+    "mdmsu": "sega-megadrive",
+    "ms": "sega-mastersystem",
+    "mark3": "sega-mastersystem",
+    "gamegearh": "sega-gamegear",
+    "megacdjp": "sega-megacd",
+    "saturnjp": "sega-saturn",
+    "sega32xjp": "sega-32x",
+    "sega32xna": "sega-32x",
+    "thirtytwox": "sega-32x",
+    "segasgone": "sega-sg1000",
+    "ps": "sony-playstation",
+    "pce": "nec-pc-engine",
+    "tg16cd": "nec-pc-engine-cd",
+    "tg-cd": "nec-pc-engine-cd",
+    "turbografxcd": "nec-pc-engine-cd",
+    "neocd": "snk-neogeo-cd",
+    "ngcd": "snk-neogeo-cd",
+    "neogeocdjp": "snk-neogeo-cd",
+    "supervision": "watara-supervision",
+    "cplus4": "commodore-plus4",
+    "cpet": "commodore-pet",
+    "cpc": "amstrad-cpc",
+    "to8": "thomson-to8",
+    "tic": "tic80",
+    "fba": "arcade",
+    "hbmame": "arcade",
+    "advmame": "arcade",
+    "mame-advmame": "arcade",
+    "mame2000": "arcade",
+    "mame2003": "arcade",
+    "mame2003plus": "arcade",
+    "mame2010": "arcade",
+    "mame-libretro": "arcade",
+    "mame-mame4all": "arcade",
+    "c20": "commodore-vic20",
+    "segamodel3": "sega-model3",
+    "pspminis": "sony-psp",
+    "ndsiware": "nintendo-dsi",
+    "wiiware": "nintendo-wii",
+    "x16": "commanderx16",
+    "zxnext": "sinclair-zx-spectrum-next",
+    "vemulator": "sega-vmu",
+    "ep64-128": "enterprise-64-128",
+    "easyrpg": "rpgmaker",
+    "ggh": "sega-gamegear",
+    "gba2players": "nintendo-gba",
+    "snesmsu": "nintendo-snes",
+    "megadrivemsu": "sega-megadrive",
+    "sgb-msu1": "nintendo-sgb",
 }
 
 

scripts/game_rules.py

@@ -0,0 +1,273 @@
+"""Load and check games/regions.yml and games/extras.yml against the profiles.
+
+    python scripts/game_rules.py            # check, print a summary
+    python scripts/game_rules.py --json     # the summary as JSON
+
+Both files name profiles and the files those profiles declare. A rule is
+only as good as that link: a profile renamed, a file dropped from it, a
+region slug the vocabulary does not know, a citation at a revision the
+profile is not pinned to, or a header trigger asking a reader of
+games/headers.yml for a value it never returns, and the installer would act
+on a rule nothing backs. Each of those is an error here.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from common import load_emulator_profiles
+from region import REGIONS, normalize_declared
+from safeparse import yaml_load
+
+REGIONS_FILE = Path("games") / "regions.yml"
+EXTRAS_FILE = Path("games") / "extras.yml"
+HEADERS_FILE = Path("games") / "headers.yml"
+
+_REF_REPO = re.compile(r"(?:^|; )([\w.-]+/[\w.-]+)@([0-9a-f]{8,40}) ")
+
+
+def load_yaml(path: Path) -> dict:
+    with Path(path).open(encoding="utf-8") as handle:
+        return yaml_load(handle) or {}
+
+
+def profile_pins(profile: dict) -> set[str]:
+    """Every commit a profile is pinned to, a build-mode dict included."""
+    pins: set[str] = set()
+    for key in ("source_commit", "upstream_commit"):
+        value = profile.get(key)
+        values = value.values() if isinstance(value, dict) else [value]
+        pins.update(str(v) for v in values if v)
+    return pins
+
+
+def declared_names(profile: dict) -> set[str]:
+    """The names and aliases a profile's files answer to."""
+    names: set[str] = set()
+    for entry in profile.get("files") or []:
+        if isinstance(entry, dict) and entry.get("name"):
+            names.add(str(entry["name"]))
+            names.update(str(a) for a in entry.get("aliases") or [])
+    return names
+
+
+def tagged_regions(profile: dict, name: str) -> set[str]:
+    """The regions a profile tags a file with, over all its declarations."""
+    out: set[str] = set()
+    for entry in profile.get("files") or []:
+        if isinstance(entry, dict) and (
+            entry.get("name") == name or name in (entry.get("aliases") or [])
+        ):
+            out |= normalize_declared(entry.get("region"))
+    return out
+
+
+def every_tag(profile: dict) -> set[str]:
+    return {
+        region
+        for entry in profile.get("files") or []
+        if isinstance(entry, dict)
+        for region in normalize_declared(entry.get("region"))
+    }
+
+
+def header_readers(headers: dict) -> dict[str, dict]:
+    """Every reader of games/headers.yml by id, over all its systems."""
+    readers: dict[str, dict] = {}
+    for system in (headers.get("systems") or {}).values():
+        readers.update(system.get("readers") or {})
+    return readers
+
+
+def reader_answers(reader: dict, field: str | None = None) -> set[object] | None:
+    """The labels and values a reader can return, None when any string can
+    be (a serial or product id read as it stands)."""
+    if field:
+        spec = (reader.get("fields") or {}).get(field) or {}
+        return {v for k, v in spec.items() if k not in ("encoding",) and isinstance(v, str)}
+    decode = reader.get("decode") or {}
+    if decode.get("kind") == "serial":
+        return None
+    answers: set[object] = set()
+    for row in decode.get("values") or []:
+        answers.update(row[k] for k in ("value", "label") if k in row)
+    for key in ("then", "otherwise"):
+        branch = decode.get(key)
+        if isinstance(branch, dict) and "label" in branch:
+            answers.add(branch["label"])
+        elif isinstance(branch, str):
+            answers.add(branch)
+    return answers
+
+
+@dataclass
+class Checker:
+    profiles: dict[str, dict]
+    readers: dict[str, dict] | None = None
+
+    def __post_init__(self) -> None:
+        self.errors: list[str] = []
+
+    def _profile(self, where: str, name: str, system: str) -> dict | None:
+        profile = self.profiles.get(name)
+        if profile is None:
+            self.errors.append(f"{where}: no profile {name}")
+            return None
+        if system not in (profile.get("systems") or []):
+            self.errors.append(f"{where}: {name} does not declare system {system}")
+        return profile
+
+    def _files(self, where: str, name: str, profile: dict, files: list[str]) -> None:
+        known = declared_names(profile)
+        for file in files:
+            if file not in known:
+                self.errors.append(f"{where}: {name} declares no file {file}")
+
+    def _region(self, where: str, value: str) -> None:
+        if value not in REGIONS:
+            self.errors.append(f"{where}: unknown region {value}")
+
+    def _ref(self, where: str, profile: dict, ref: str | None) -> None:
+        if not ref:
+            self.errors.append(f"{where}: no source_ref")
+            return
+        pins = profile_pins(profile)
+        cited = _REF_REPO.findall(ref)
+        if not cited:
+            self.errors.append(f"{where}: source_ref names no owner/repo@sha")
+        for repo, sha in cited:
+            if not any(pin.startswith(sha) for pin in pins):
+                self.errors.append(f"{where}: {repo}@{sha} is not a pin of the profile")
+
+    def regions(self, doc: dict) -> None:
+        cases = doc.get("cases") or {}
+        for system, cores in (doc.get("systems") or {}).items():
+            for name, rule in cores.items():
+                where = f"regions {system}/{name}"
+                profile = self._profile(where, name, system)
+                if profile is None:
+                    continue
+                self._ref(where, profile, rule.get("source_ref"))
+                tags = every_tag(profile)
+                for src, dst in (rule.get("routes") or {}).items():
+                    self._region(where, src)
+                    self._region(where, dst)
+                    if dst not in tags:
+                        self.errors.append(f"{where}: route {src} -> {dst}, but no file is tagged {dst}")
+                for value in [rule.get("default_region"), *(rule.get("preference") or [])]:
+                    if value:
+                        self._region(where, value)
+                free = rule.get("region_free") or []
+                self._files(where, name, profile, free)
+                for file in free:
+                    specific = tagged_regions(profile, file) - {"world"}
+                    if specific:
+                        self.errors.append(
+                            f"{where}: region_free {file} is tagged {sorted(specific)} in the profile"
+                        )
+                for keep in rule.get("keep_all_regions") or []:
+                    kwhere = f"{where} case {keep.get('case')}"
+                    if keep.get("case") not in cases:
+                        self.errors.append(f"{kwhere}: case not described under cases")
+                    self._ref(kwhere, profile, keep.get("source_ref"))
+                    self._files(kwhere, name, profile, keep.get("files") or [])
+
+    def extras(self, doc: dict) -> None:
+        for index, entry in enumerate(doc.get("extras") or []):
+            where = f"extras[{index}] {entry.get('system')}/{entry.get('profile')}"
+            profile = self._profile(where, entry["profile"], entry["system"])
+            if profile is None:
+                continue
+            self._ref(where, profile, entry.get("source_ref"))
+            self._files(where, entry["profile"], profile, entry.get("files") or [])
+            header = (entry.get("trigger") or {}).get("header")
+            if header and self.readers is not None:
+                self._header(where, header)
+
+    def _header(self, where: str, header: dict) -> None:
+        reader = self.readers.get(header["reader"])
+        if reader is None:
+            self.errors.append(f"{where}: games/headers.yml has no reader {header['reader']}")
+            return
+        field = header.get("field")
+        if field and field not in (reader.get("fields") or {}):
+            self.errors.append(f"{where}: reader {header['reader']} has no field {field}")
+            return
+        answers = reader_answers(reader, field)
+        if answers is None:
+            return
+        for value in header.get("values") or []:
+            if value not in answers:
+                self.errors.append(
+                    f"{where}: reader {header['reader']} never returns {value!r}"
+                )
+
+
+def check(
+    regions: dict, extras: dict, profiles: dict[str, dict], headers: dict | None = None
+) -> list[str]:
+    """Errors over both files. Without headers, header triggers go unchecked."""
+    checker = Checker(profiles, header_readers(headers) if headers is not None else None)
+    checker.regions(regions)
+    checker.extras(extras)
+    return checker.errors
+
+
+def summary(regions: dict, extras: dict) -> dict:
+    keeps = [
+        {"case": keep["case"], "system": system, "profile": name, "condition": keep["condition"]}
+        for system, cores in sorted((regions.get("systems") or {}).items())
+        for name, rule in sorted(cores.items())
+        for keep in rule.get("keep_all_regions") or []
+    ]
+    systems = regions.get("systems") or {}
+    return {
+        "region_systems": len(systems),
+        "region_cores": sum(len(c) for c in systems.values()),
+        "cases": sorted(regions.get("cases") or {}),
+        "case_entries": keeps,
+        "extras": len(extras.get("extras") or []),
+        "extra_systems": sorted({e["system"] for e in extras.get("extras") or []}),
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
+    parser.add_argument("--regions", type=Path, default=REGIONS_FILE,
+                        help=f"region rules to check (default {REGIONS_FILE})")
+    parser.add_argument("--extras", type=Path, default=EXTRAS_FILE,
+                        help=f"extra-file rules to check (default {EXTRAS_FILE})")
+    parser.add_argument("--headers", type=Path, default=HEADERS_FILE,
+                        help=f"header readers the triggers name (default {HEADERS_FILE})")
+    parser.add_argument("--emulators-dir", default="emulators",
+                        help="profiles the rules are checked against (default emulators)")
+    parser.add_argument("--json", action="store_true", help="print the summary as JSON")
+    args = parser.parse_args()
+
+    regions = load_yaml(args.regions)
+    extras = load_yaml(args.extras)
+    errors = check(regions, extras, load_emulator_profiles(args.emulators_dir), load_yaml(args.headers))
+    info = summary(regions, extras)
+    if args.json:
+        print(json.dumps({**info, "errors": errors}, indent=2))
+    else:
+        print(
+            f"Region rules: {info['region_cores']} cores over {info['region_systems']} systems, "
+            f"{len(info['cases'])} cases in {len(info['case_entries'])} entries"
+        )
+        for row in info["case_entries"]:
+            print(f"  {row['case']:26} {row['system']:20} {row['profile']:22} {row['condition']}")
+        print(f"Per-game extras: {info['extras']} entries over {len(info['extra_systems'])} systems")
+        for error in errors:
+            print(f"ERROR {error}")
+    return 1 if errors else 0
+
+
+if __name__ == "__main__":
+    from common import run_cli
+
+    run_cli(main)

scripts/generate_event_schema.py

@@ -43,11 +43,12 @@ KEY_TYPES: dict[str, dict] = {
         "files", "files_done", "files_total", "bytes", "bytes_done", "bytes_total",
         "bytes_local", "bytes_network", "size", "present", "missing", "wrong_hash",
         "bytes_needed", "data_missing", "not_installable", "installed", "up_to_date",
-        "pending", "code",
+        "pending", "code", "games", "recognised", "not_games",
     ), _COUNT),
-    **dict.fromkeys(("truncated", "ok", "interrupted"), _FLAG),
+    **dict.fromkeys(("truncated", "ok", "interrupted", "required_only"), _FLAG),
     **dict.fromkeys(("systems", "other_drives", "dests", "written"), _NAMES),
-    **dict.fromkeys(("platforms", "failed", "skipped", "left_out", "kept"), _RECORDS),
+    **dict.fromkeys(("platforms", "failed", "skipped", "left_out", "kept", "found",
+                     "fallback"), _RECORDS),
 }
 
 

scripts/generate_games_index.py

@@ -0,0 +1,620 @@
+"""Publish what the installer reads to install for the games on a card.
+
+The installer's "For the games on this card" mode reads the card's rom
+folders, identifies each game and installs only the BIOS those games need.
+This step joins the sources that answer each part and writes them next to the
+install lists, under install/games/:
+
+- index.json: format version, a digest of the data, and every published
+  file with its size, its gzip size (raw.githubusercontent.com compresses on
+  request) and its sha256, so a phone fetches only what its card needs.
+- setups/<setup>.json: the setup's rom folders (platforms/romfolders/),
+  resolved to repository systems and profiles, with the key of each folder's
+  fallback set.
+- fallback/<platform>.json: per system and default core, the files the
+  platform's install list carries for them (computed from install/<platform>.json
+  with the installer's own core narrowing), their sizes and whether each is
+  required. What an unrecognised game keeps.
+- systems/<system>.json: identification (games/catalogs/, crc32 + size to a
+  region and a title, disc serials to a region), header readers
+  (games/headers.yml), region rules per profile (games/regions.yml joined with
+  the profiles' region: fields) and per-game extra files (games/extras.yml).
+- arcade/<profile>.json: game set to BIOS parent and device sets
+  (games/arcade/).
+
+Nothing here is written by hand: a second run writes the same bytes.
+
+    python scripts/generate_games_index.py
+    python scripts/generate_games_index.py --check   # exit 1 if a file would change
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import hashlib
+import json
+import sys
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from common import (  # noqa: E402
+    list_registered_platforms,
+    load_emulator_profiles,
+    load_platform_config,
+    native_layer,
+    run_cli,
+)
+from region import normalize_declared  # noqa: E402
+from romfolders import (  # noqa: E402
+    ProfileResolver,
+    Resolution,
+    SystemResolver,
+    folder_names,
+    load_setups,
+)
+from safeparse import yaml_load  # noqa: E402
+
+import install  # noqa: E402
+
+FORMAT_VERSION = 1
+OUTPUT = Path("install") / "games"
+GAMES = Path("games")
+
+# Coverage the published data may not fall below. A rescrape or a DAT import
+# that loses rows fails the step instead of shipping a thinner index.
+FLOORS = {
+    "folders_resolved_share": 0.84,
+    "catalog_systems": 30,
+    "arcade_profiles": 6,
+}
+
+
+@dataclass(frozen=True)
+class Inputs:
+    """Where each source is read from. The defaults are the repository's."""
+
+    games: Path = GAMES
+    romfolders: Path = Path("platforms") / "romfolders"
+    install: Path = Path("install")
+    emulators: str = "emulators"
+    platforms: str = "platforms"
+
+
+def dumps(document: object) -> str:
+    return json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n"
+
+
+def _load_yaml(path: Path) -> dict:
+    if not path.exists():
+        return {}
+    with path.open(encoding="utf-8") as handle:
+        return yaml_load(handle) or {}
+
+
+def _load_json(path: Path) -> dict:
+    with path.open(encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+# --- Profiles ---------------------------------------------------------------
+
+
+def _entry_systems(profile: dict, entry: dict) -> set[str]:
+    """The systems a profile file serves: its own system field, else the profile's."""
+    own = entry.get("system") or entry.get("systems")
+    if own:
+        return {own} if isinstance(own, str) else {str(s) for s in own}
+    return {str(s) for s in profile.get("systems") or []}
+
+
+def _entry_names(entry: dict) -> set[str]:
+    names = {str(entry.get("name") or "")}
+    names |= {str(a) for a in entry.get("aliases") or []}
+    for key in ("path", "standalone_path", "destination"):
+        if entry.get(key):
+            names.add(Path(str(entry[key])).name)
+    return {n.lower() for n in names if n}
+
+
+class Profiles:
+    """Profile lookups the fallback and region sections share."""
+
+    def __init__(self, profiles: dict[str, dict], resolver: SystemResolver) -> None:
+        self.profiles = profiles
+        self.resolver = resolver
+        self._by_name: dict[str, dict[str, list[dict]]] = {}
+
+    def entries_named(self, profile: str, name: str) -> list[dict]:
+        table = self._by_name.get(profile)
+        if table is None:
+            table = {}
+            for entry in self.profiles.get(profile, {}).get("files") or []:
+                if isinstance(entry, dict):
+                    for n in _entry_names(entry):
+                        table.setdefault(n, []).append(entry)
+            self._by_name[profile] = table
+        return table.get(name.lower(), [])
+
+    def serves(self, profile: str, entry: dict, system: str) -> bool:
+        wanted = self.resolver.equivalents(system)
+        return bool(_entry_systems(self.profiles.get(profile, {}), entry) & wanted)
+
+    def region_files(self, profile: str, system: str) -> dict[str, list[str]]:
+        """Region slug -> file names the profile tags for it, for one system.
+
+        `any` holds the files declared with no region: those are installed
+        whatever the game's region.
+        """
+        out: dict[str, set[str]] = {}
+        data = self.profiles.get(profile, {})
+        for entry in data.get("files") or []:
+            if not isinstance(entry, dict) or not self.serves(profile, entry, system):
+                continue
+            if entry.get("category") == "game_data":
+                continue
+            regions = normalize_declared(entry.get("region")) or {"any"}
+            for region in regions:
+                out.setdefault(region, set()).add(str(entry["name"]))
+        return {k: sorted(v) for k, v in sorted(out.items())}
+
+
+# --- Fallback sets ----------------------------------------------------------
+
+
+def _platform_required(platform: str, platforms_dir: str) -> dict[tuple[str, str], bool]:
+    """(system, file name) -> required, as the platform's own list declares it."""
+    config = load_platform_config(platform, platforms_dir)
+    out: dict[tuple[str, str], bool] = {}
+    for sys_id, system in (native_layer(config) or config).get("systems", {}).items():
+        for entry in system.get("files", []):
+            for name in {entry.get("name"), Path(str(entry.get("destination") or "")).name} - {None, ""}:
+                out[(sys_id, str(name).lower())] = entry.get("required") is not False
+    return out
+
+
+def fallback_set(
+    manifest: dict,
+    platform_required: dict[tuple[str, str], bool],
+    profiles: Profiles,
+    system: str,
+    profile: str | None,
+) -> dict:
+    """The files an unrecognised game of one system keeps on one platform.
+
+    The platform's own files for the system, plus what the default core's
+    profile reads for it. The core step is the installer's own narrowing
+    (`install.narrow` with the designated cores). A core file the list does
+    not attribute to a platform system is kept when the profile entry that
+    declares it serves the system.
+    """
+    wanted = profiles.resolver.equivalents(system)
+    cores = sorted(install.designated_cores(manifest, [profile])) if profile else []
+    files, _omitted = install.narrow(manifest["files"], manifest.get("omitted_files", []), [], cores, [])
+    rows: list[list] = []
+    for entry in files:
+        systems = set(entry.get("systems") or [])
+        dest = entry["dest"]
+        base = Path(dest).name.lower()
+        if entry.get("cores") is None:
+            if not systems & wanted:
+                continue
+            required = any(
+                platform_required.get((s, base), True) for s in systems & wanted
+            )
+        else:
+            if systems:
+                if not systems & wanted:
+                    continue
+            declared = profiles.entries_named(profile or "", base)
+            if not systems and not any(profiles.serves(profile or "", e, system) for e in declared):
+                continue
+            required = any(e.get("required") is not False for e in declared) if declared else True
+        rows.append([dest, int(entry["size"]), required, entry.get("category") or "bios"])
+    rows.sort()
+    return {
+        "files": rows,
+        "bytes": sum(r[1] for r in rows),
+        "required_bytes": sum(r[1] for r in rows if r[2]),
+    }
+
+
+def _shared_table(platform: str, sets: dict[str, dict]) -> dict:
+    """One row per install-list file, and each set as indices into it.
+
+    The sets of the MAME-run systems hold the same thousand files: written
+    out per set, Batocera's table weighed half a megabyte compressed.
+    """
+    rows = sorted({(r[0], r[1], r[3]) for body in sets.values() for r in body["files"]})
+    position = {row: i for i, row in enumerate(rows)}
+    published = {}
+    for key, body in sorted(sets.items()):
+        published[key] = {
+            "files": [position[(r[0], r[1], r[3])] for r in body["files"]],
+            "optional": [position[(r[0], r[1], r[3])] for r in body["files"] if not r[2]],
+            "bytes": body["bytes"],
+            "required_bytes": body["required_bytes"],
+        }
+    return {
+        "format_version": FORMAT_VERSION,
+        "platform": platform,
+        "files": [list(r) for r in rows],
+        "sets": published,
+    }
+
+
+# --- Setups -----------------------------------------------------------------
+
+
+def _arcade_profiles(directory: Path) -> set[str]:
+    return {p.stem for p in directory.glob("*.json")} if directory.is_dir() else set()
+
+
+def resolve_folder(
+    row: dict, platform: str, profile: str | None, resolver: SystemResolver, profiles: Profiles
+) -> Resolution:
+    """The systems a folder holds: its names first, else its core's one machine.
+
+    A name nothing spells, run by a core of one machine, is that machine
+    (ECWolf's wolf3d folder, a port's own folder).
+    """
+    resolution = resolver.resolve(folder_names(row), platform)
+    own = [str(s) for s in profiles.profiles.get(profile or "", {}).get("systems") or []]
+    if not resolution.systems and len(own) == 1:
+        return Resolution([resolver.canonical(own[0])], f"only system of {profile}")
+    return resolution
+
+
+def build_setups(
+    setups: dict[str, dict],
+    resolver: SystemResolver,
+    cores: ProfileResolver,
+    profiles: Profiles,
+    arcade: set[str],
+    inputs: Inputs,
+) -> tuple[dict[str, dict], dict[str, dict], dict]:
+    """Published setup documents, fallback tables per platform, coverage."""
+    manifests: dict[str, dict] = {}
+    required: dict[str, dict] = {}
+    fallbacks: dict[str, dict[str, dict]] = {}
+    out: dict[str, dict] = {}
+    coverage: dict[str, dict] = {}
+    for setup, doc in sorted(setups.items()):
+        platform = doc.get("platform") or ""
+        if platform and platform not in manifests:
+            path = inputs.install / f"{platform}.json"
+            if path.exists():
+                manifests[platform] = _load_json(path)
+                required[platform] = _platform_required(platform, inputs.platforms)
+        manifest = manifests.get(platform)
+        folders: list[dict] = []
+        counts = {"folders": 0, "resolved": 0, "profile": 0, "fallback": 0}
+        for row in doc.get("folders", []):
+            counts["folders"] += 1
+            profile = cores.resolve(row.get("core"))
+            resolution = resolve_folder(row, platform, profile, resolver, profiles)
+            published: dict = {
+                "folder": row["folder"],
+                "systems": resolution.systems,
+                "basis": resolution.basis,
+            }
+            for key in ("name", "aliases", "extensions", "core", "emulator", "note"):
+                if row.get(key):
+                    published[key] = row[key]
+            if profile:
+                published["profile"] = profile
+                counts["profile"] += 1
+            if resolution.systems:
+                counts["resolved"] += 1
+
+            def fallback_keys(chosen: str | None) -> list[str]:
+                """The fallback sets of the folder's systems under one core,
+                computed once per platform."""
+                keys = []
+                table = fallbacks.setdefault(platform, {})
+                for system in resolution.systems:
+                    key = f"{system}|{chosen or ''}"
+                    if key not in table:
+                        table[key] = fallback_set(manifest, required[platform], profiles, system, chosen)
+                    keys.append(key)
+                return keys
+
+            sets_known = manifest is not None and bool(resolution.systems)
+            boards = {}
+            for board, spec in sorted((row.get("board_defaults") or {}).items()):
+                boards[board] = {"core": spec["core"]}
+                board_profile = cores.resolve(spec["core"])
+                if board_profile:
+                    boards[board]["profile"] = board_profile
+                    if board_profile in arcade:
+                        boards[board]["arcade"] = board_profile
+                if sets_known:
+                    # A card moves between boards, and the installer reads
+                    # it on a computer: each board's core has its set.
+                    boards[board]["fallback"] = fallback_keys(board_profile)
+            if boards:
+                published["board_defaults"] = boards
+            if profile and profile in arcade:
+                published["arcade"] = profile
+            elif row.get("arcade"):
+                published["arcade"] = True
+            if sets_known:
+                keys = fallback_keys(profile)
+                published["fallback"] = keys
+                if any(fallbacks[platform][k]["files"] for k in keys):
+                    counts["fallback"] += 1
+            published["source_ref"] = row["source_ref"]
+            folders.append(published)
+        out[setup] = {
+            "format_version": FORMAT_VERSION,
+            "setup": setup,
+            "name": doc["name"],
+            "platform": platform,
+            "sources": doc["sources"],
+            "roms_root": doc["roms_root"],
+            "case_sensitive": doc["case_sensitive"],
+            **{k: doc[k] for k in ("skip", "hidden", "media", "notes") if doc.get(k)},
+            "folders": folders,
+        }
+        coverage[setup] = counts
+    published_fallbacks = {
+        platform: _shared_table(platform, table) for platform, table in sorted(fallbacks.items())
+    }
+    return out, published_fallbacks, coverage
+
+
+# --- Systems ----------------------------------------------------------------
+
+
+def _catalogs(directory: Path) -> Iterable[dict]:
+    if directory.is_dir():
+        for path in sorted(directory.glob("*/*.json")):
+            yield _load_json(path)
+
+
+def build_systems(
+    resolver: SystemResolver,
+    profiles: Profiles,
+    used: dict[str, set[str]],
+    games: Path,
+) -> tuple[dict[str, dict], dict]:
+    """Published system documents and per-system coverage.
+
+    `used` holds, per system, the profiles some setup starts for it: their
+    region files are published even without a rule of their own.
+    """
+    headers = _load_yaml(games / "headers.yml").get("systems", {})
+    extras: dict[str, list[dict]] = {}
+    for entry in _load_yaml(games / "extras.yml").get("extras", []):
+        extras.setdefault(entry["system"], []).append(entry)
+    rule_file = _load_yaml(games / "regions.yml")
+    cases = rule_file.get("cases", {})
+    docs: dict[str, dict] = {}
+
+    def doc_for(system: str) -> dict:
+        key = resolver.canonical(system)
+        return docs.setdefault(key, {"format_version": FORMAT_VERSION, "system": key})
+
+    for catalog in _catalogs(games / "catalogs"):
+        doc = doc_for(catalog["system"])
+        regions_table = catalog["regions"]
+        fields = catalog["fields"]
+        col = {name: i for i, name in enumerate(fields)}
+        ident = doc.setdefault("identify", {"sources": [], "regions": [], "games": [], "serials": {}})
+        region_index: dict[tuple[str, ...], int] = {
+            tuple(r): i for i, r in enumerate(ident["regions"])
+        }
+
+        def reindex(i: int) -> int:
+            value = tuple(regions_table[i]) if i is not None and i >= 0 else ()
+            if value not in region_index:
+                region_index[value] = len(ident["regions"])
+                ident["regions"].append(list(value))
+            return region_index[value]
+
+        ident["sources"].append({
+            k: catalog[k] for k in ("source", "dat", "version", "fetched_from", "licence", "serial_note") if k in catalog
+        })
+        for row in catalog["games"]:
+            region = reindex(row[col["region"]])
+            ident["games"].append([row[col["crc32"]], row[col["size"]], region, row[col["title"]]])
+            for serial in row[col["serials"]] if "serials" in col else []:
+                ident["serials"].setdefault(serial, set()).add(region)
+    for doc in docs.values():
+        ident = doc.get("identify")
+        if ident:
+            ident["games"].sort()
+            ident["serials"] = {s: sorted(r) for s, r in sorted(ident["serials"].items())}
+    for system, spec in headers.items():
+        doc_for(system)["headers"] = spec
+    for system, entries in extras.items():
+        doc_for(system).setdefault("extras", []).extend(entries)
+    rule_profiles: dict[str, set[str]] = {}
+    for system, spec in rule_file.get("systems", {}).items():
+        doc = doc_for(system)
+        rules = doc.setdefault("region_rules", {})
+        rules.update(spec)
+        named = {
+            case["case"] for core in spec.values()
+            for case in (core.get("keep_all_regions") or []) if isinstance(case, dict)
+        }
+        if named:
+            doc.setdefault("cases", {}).update({c: cases[c] for c in sorted(named) if c in cases})
+        rule_profiles.setdefault(doc["system"], set()).update(spec if isinstance(spec, dict) else ())
+    for system, names in used.items():
+        rule_profiles.setdefault(resolver.canonical(system), set()).update(names)
+    for system, names in rule_profiles.items():
+        files = {
+            name: tagged for name in sorted(names)
+            if name in profiles.profiles
+            and (tagged := profiles.region_files(name, system))
+            and set(tagged) - {"any"}
+        }
+        if files:
+            doc_for(system)["region_files"] = files
+    coverage = {
+        system: {
+            "games": len(doc.get("identify", {}).get("games", [])),
+            "serials": len(doc.get("identify", {}).get("serials", {})),
+            "header_readers": len((doc.get("headers") or {}).get("readers", {}) or []),
+            "region_profiles": len(doc.get("region_files", {})),
+            "extras": bool(doc.get("extras")),
+        }
+        for system, doc in sorted(docs.items())
+    }
+    return docs, coverage
+
+
+# --- Arcade -----------------------------------------------------------------
+
+
+def build_arcade(directory: Path) -> dict[str, dict]:
+    if not directory.is_dir():
+        return {}
+    return {path.stem: _load_json(path) for path in sorted(directory.glob("*.json"))}
+
+
+# --- Writing ----------------------------------------------------------------
+
+
+def _entry(text: str) -> dict:
+    data = text.encode("utf-8")
+    return {
+        "size": len(data),
+        "gzip_size": len(gzip.compress(data, compresslevel=9, mtime=0)),
+        "sha256": hashlib.sha256(data).hexdigest(),
+    }
+
+
+def folder_systems(setups: dict[str, dict]) -> dict[str, list[str]]:
+    """Every folder name a setup gives a system, casefolded, with the systems
+    any setup reads there, and each system id under its own name.
+
+    What a games folder on a computer is read with when its setup names no
+    folder of its own (RetroArch): a name two setups give two systems keeps
+    both.
+    """
+    names: dict[str, set[str]] = {}
+    for doc in setups.values():
+        for row in doc["folders"]:
+            systems = row["systems"]
+            if not systems:
+                continue
+            for name in [row["folder"], *(row.get("aliases") or [])]:
+                names.setdefault(str(name).casefold(), set()).update(systems)
+            for system in systems:
+                names.setdefault(system.casefold(), set()).add(system)
+    return {name: sorted(systems) for name, systems in sorted(names.items())}
+
+
+def render(output: Path = OUTPUT, inputs: Inputs = Inputs()) -> tuple[dict[Path, str], dict]:
+    """Every published file's text, keyed by its path, and the coverage."""
+    resolver = SystemResolver.from_repo(inputs.emulators, inputs.platforms)
+    profiles = Profiles(load_emulator_profiles(inputs.emulators), resolver)
+    arcade = build_arcade(inputs.games / "arcade")
+    setups, fallbacks, setup_cov = build_setups(
+        load_setups(inputs.romfolders), resolver, ProfileResolver.from_repo(inputs.emulators),
+        profiles, set(arcade), inputs,
+    )
+    used: dict[str, set[str]] = {}
+    for doc in setups.values():
+        for row in doc["folders"]:
+            for system in row["systems"]:
+                if row.get("profile"):
+                    used.setdefault(system, set()).add(row["profile"])
+    systems, system_cov = build_systems(resolver, profiles, used, inputs.games)
+
+    files: dict[Path, str] = {}
+    index: dict = {"format_version": FORMAT_VERSION, "setups": {}, "fallback": {}, "systems": {}, "arcade": {}}
+    for kind, docs in (("setups", setups), ("fallback", fallbacks), ("systems", systems), ("arcade", arcade)):
+        for key, doc in sorted(docs.items()):
+            rel = Path(kind) / f"{key}.json"
+            text = dumps(doc)
+            files[output / rel] = text
+            index[kind][key] = {"file": rel.as_posix(), **_entry(text)}
+    digest = hashlib.sha256(
+        "".join(f"{p.relative_to(output).as_posix()} {_entry(t)['sha256']}\n" for p, t in sorted(files.items())).encode()
+    ).hexdigest()
+    index["data_version"] = digest[:16]
+    index["licences"] = sorted(
+        {json.dumps(src.get("licence"), sort_keys=True)
+         for doc in systems.values() for src in doc.get("identify", {}).get("sources", [])
+         if src.get("licence")}
+    )
+    index["licences"] = [json.loads(x) for x in index["licences"]]
+    containers = _load_yaml(inputs.games / "headers.yml").get("containers")
+    if containers:
+        index["containers"] = containers
+    index["folders"] = folder_systems(setups)
+    files[output / "index.json"] = dumps(index)
+    coverage = {"setups": setup_cov, "systems": system_cov, "arcade": sorted(arcade)}
+    return files, coverage
+
+
+def floor_failures(coverage: dict) -> list[str]:
+    total = sum(c["folders"] for c in coverage["setups"].values())
+    resolved = sum(c["resolved"] for c in coverage["setups"].values())
+    out = []
+    share = resolved / total if total else 0.0
+    if share < FLOORS["folders_resolved_share"]:
+        out.append(f"folders resolved {resolved}/{total} ({share:.1%}) under {FLOORS['folders_resolved_share']:.0%}")
+    catalogs = sum(1 for c in coverage["systems"].values() if c["games"])
+    if catalogs < FLOORS["catalog_systems"]:
+        out.append(f"systems with a game catalog {catalogs} under {FLOORS['catalog_systems']}")
+    if len(coverage["arcade"]) < FLOORS["arcade_profiles"]:
+        out.append(f"arcade profiles {len(coverage['arcade'])} under {FLOORS['arcade_profiles']}")
+    return out
+
+
+def print_coverage(coverage: dict, files: dict[Path, str]) -> None:
+    print("Setups (folders / resolved to a system / default core profiled / with a fallback set):")
+    for setup, c in coverage["setups"].items():
+        print(f"  {setup:16} {c['folders']:4} {c['resolved']:4} {c['profile']:4} {c['fallback']:4}")
+    print("Systems (games by hash / serials / header readers / profiles with region files / extras):")
+    for system, c in coverage["systems"].items():
+        print(f"  {system:28} {c['games']:6} {c['serials']:6} {c['header_readers']:3} "
+              f"{c['region_profiles']:3} {'yes' if c['extras'] else '-'}")
+    print(f"Arcade indexes: {', '.join(coverage['arcade']) or 'none'}")
+    raw = sum(len(t.encode()) for t in files.values())
+    packed = sum(len(gzip.compress(t.encode(), 9, mtime=0)) for t in files.values())
+    print(f"{len(files)} files, {raw:,} bytes, {packed:,} bytes gzip")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
+    parser.add_argument("--output-dir", type=Path, default=OUTPUT,
+                        help=f"where the index is written (default {OUTPUT})")
+    parser.add_argument("--check", action="store_true", help="exit 1 when a file would change")
+    parser.add_argument("--quiet", action="store_true", help="print only failures")
+    args = parser.parse_args()
+
+    files, coverage = render(args.output_dir)
+    if not args.quiet:
+        print_coverage(coverage, files)
+    failures = floor_failures(coverage)
+    for failure in failures:
+        print(f"FLOOR {failure}", file=sys.stderr)
+    stale = sorted(
+        p for p in args.output_dir.rglob("*.json") if p not in files
+    ) if args.output_dir.is_dir() else []
+    changed = [p for p, t in files.items() if not p.exists() or p.read_text(encoding="utf-8") != t]
+    if args.check:
+        for path in changed + stale:
+            print(f"STALE {path}", file=sys.stderr)
+        return 1 if changed or stale or failures else 0
+    for path in stale:
+        path.unlink()
+    for path in changed:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(files[path], encoding="utf-8")
+    print(f"{len(changed)} written, {len(stale)} removed")
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    run_cli(main)

scripts/generate_installer_doc.py

@@ -14,12 +14,15 @@ install.py gains has no prose yet: the generator names it and writes nothing,
 and the row is added by hand inside the markers, its first cell holding the
 option, kind, code or value alone.
 
-The run samples (a dry run, a first install, a later check) are printed by
-install.py itself, run in its own process against the committed
-install/<platform>.json served on the loopback, from a home and a card made
-under tmp/ and shown as /home/me and /userdata. Only the transport and the
-machine are stood in for, see _child. Each is dated by the `generated` field
-of its list, so a rendering is the same from one day to the next.
+The run samples (a dry run, a first install, a later check, the reasons a
+file is refused, a run copying from an old card) are printed by install.py
+itself, run in its own process against the committed install/<platform>.json
+served on the loopback, from a home and a card made under tmp/ and shown as
+/home/me and /userdata. The reasons and the copy are read from a four-file
+list laid out in that home (SMALL_LIST), whose files are written there. Only
+the transport and the machine are stood in for, see _child. Each is dated by
+the `generated` field of its list, so a rendering is the same from one day
+to the next.
 """
 
 from __future__ import annotations
@@ -33,10 +36,12 @@ import json
 import os
 import re
 import shutil
+import socketserver
 import subprocess
 import sys
 import tempfile
 import threading
+import zipfile
 from pathlib import Path, PurePosixPath
 from typing import Callable
 
@@ -282,6 +287,44 @@ class _ListHandler(http.server.SimpleHTTPRequestHandler):
         return
 
 
+class _LoopbackServer(http.server.ThreadingHTTPServer):
+    """The list server, named by its number: HTTPServer.server_bind looks
+    127.0.0.1 up with socket.getfqdn, a DNS query wherever /etc/hosts does
+    not name it (code_audit_24 P3-12)."""
+
+    def server_bind(self) -> None:
+        socketserver.TCPServer.server_bind(self)
+        host, port = self.server_address[:2]
+        self.server_name, self.server_port = host, port
+
+
+# The list the reasons and the copy are shown with: four files, each with
+# the contents its name says, read as RetroArch's.
+SMALL_LIST = {
+    "scph5500.bin": (b"J" * 64, ["sony-playstation"], ["japan"], None),
+    "scph5501.bin": (b"U" * 64, ["sony-playstation"], ["north-america"],
+                     ["beetle_psx", "pcsx_rearmed"]),
+    "gba_bios.bin": (b"G" * 32, ["nintendo-gba"], None, ["mgba"]),
+    "pcsx/extra.bin": (b"E" * 8, ["sony-playstation"], None, ["pcsx_rearmed"]),
+}
+
+
+def write_small_list(path: Path) -> None:
+    import hashlib  # noqa: PLC0415
+
+    files = []
+    for dest, (data, systems, regions, cores) in SMALL_LIST.items():
+        files.append({"dest": dest, "size": len(data), "sha1": hashlib.sha1(data).hexdigest(),
+                      "sha256": hashlib.sha256(data).hexdigest(),
+                      "md5": hashlib.md5(data).hexdigest(), "systems": systems,
+                      "cores": cores, "repo_path": f"bios/{dest}",
+                      **({"regions": regions} if regions else {})})
+    path.write_text(json.dumps({
+        "manifest_version": 2, "platform": "retroarch", "display_name": "RetroArch",
+        "base_destination": "system", "files": files, "total_files": len(files),
+        "total_size": sum(entry["size"] for entry in files)}), encoding="utf-8")
+
+
 def _child(spec: dict) -> int:
     """install.main() as the sample runs it, in its own process. Only the
     transport and the machine are stood in for: no connection leaves the
@@ -300,8 +343,13 @@ def _child(spec: dict) -> int:
             raise OSError(f"the sample makes no request to {address[0]}")
         return real_connect(address, *args, **kwargs)
 
-    def landed(f: dict, _bios: Path, _verbose: bool = False, progress: object = None,
-               _copy: object = None) -> tuple[str, bool, str]:
+    real_download = _install._download_one
+
+    def landed(f: dict, bios: Path, verbose: bool = False, progress: object = None,
+               copy: object = None) -> tuple[str, bool, str]:
+        if copy is not None:
+            # A copy from this machine is made for real: it is local.
+            return real_download(f, bios, verbose, progress, copy)
         if progress is not None:
             progress.add(f["size"])
         return f["dest"], True, ""
@@ -353,7 +401,7 @@ class Sandbox:
         config = self.home / ".config" / "retroarch" / "retroarch.cfg"
         config.parent.mkdir(parents=True)
         config.write_text(RETROARCH_CFG, encoding="utf-8")
-        self.server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _ListHandler)
+        self.server = _LoopbackServer(("127.0.0.1", 0), _ListHandler)
         threading.Thread(target=self.server.serve_forever, daemon=True).start()
         return self
 
@@ -362,26 +410,55 @@ class Sandbox:
         self.server.server_close()
         shutil.rmtree(self.root)
 
+    def small(self, kind: str) -> str:
+        """A run over SMALL_LIST in a home of its own: "refused", a --check
+        of a folder holding a copy of each kind a check names, or "from", a
+        run copying from an old card what it holds and downloading the rest."""
+        home = self.root / kind
+        system = home / ".config" / "retroarch" / "system"
+        system.mkdir(parents=True)
+        (system.parent / "retroarch.cfg").write_text(RETROARCH_CFG, encoding="utf-8")
+        listed = home / "list.json"
+        write_small_list(listed)
+        data = {dest: entry[0] for dest, entry in SMALL_LIST.items()}
+        if kind == "refused":
+            (system / "scph5500.bin").write_bytes(data["scph5500.bin"])
+            (system / "scph5501.bin").write_bytes(data["scph5500.bin"])
+            (system / "system").mkdir()
+            (system / "system" / "gba_bios.bin").write_bytes(data["gba_bios.bin"])
+            with zipfile.ZipFile(system / "PSX BIOS.zip", "w") as archive:
+                archive.writestr("extra.bin", data["pcsx/extra.bin"])
+            # A check that flags a file ends on 1, as documented.
+            return self.run("retroarch", "--check", "--manifest", str(listed), home=home,
+                            codes=(1,))
+        old = home / "old"
+        old.mkdir()
+        for dest in ("scph5500.bin", "scph5501.bin"):
+            (old / dest).write_bytes(data[dest])
+        return self.run("retroarch", "--yes", "--manifest", str(listed), "--from", str(old),
+                        home=home, landed=True)
+
     def run(self, platform: str, *argv: str, landed: bool = False,
-            held: bool = False) -> str:
+            held: bool = False, home: Path | None = None, codes: tuple = (0,)) -> str:
         """What install.py prints for the run, the sandbox read as a user's
         home and card."""
+        home = home or self.home
         card = self.card / "userdata" / "bios" if platform != "retroarch" else None
         if card is not None:
             card.mkdir(parents=True, exist_ok=True)
         spec = {"platform": platform, "argv": ["--plain", "--platform", platform, *argv],
                 "card": str(card) if card else None, "landed": landed, "held": held}
-        env = {"PATH": os.environ.get("PATH", ""), "HOME": str(self.home),
+        env = {"PATH": os.environ.get("PATH", ""), "HOME": str(home),
                "RETROBIOS_OS": "linux", "LANG": "C.UTF-8",
                "RETROBIOS_BASE_URL": f"http://127.0.0.1:{self.server.server_address[1]}"}
         done = subprocess.run(
             [sys.executable, "-s", "-c", CHILD, json.dumps(spec)],
             cwd=ROOT, env=env, stdin=subprocess.DEVNULL, capture_output=True, text=True,
             check=False)
-        if done.returncode != 0:
+        if done.returncode not in codes:
             raise SampleFailed(f"install.py {' '.join(spec['argv'])} exited "
                                f"{done.returncode}: {done.stderr.strip()[-2000:]}")
-        return done.stdout.replace(str(self.home), SHOWN_HOME).replace(str(self.card), SHOWN_CARD)
+        return done.stdout.replace(str(home), SHOWN_HOME).replace(str(self.card), SHOWN_CARD)
 
 
 # The runs printed in this process, by what they read: the page is rendered
@@ -411,6 +488,14 @@ class _LazySandbox:
         _RUNS[key] = self.box.run(platform, *argv, **kinds)
         return _RUNS[key]
 
+    def small(self, kind: str) -> str:
+        key = ("small", kind, _stamp("retroarch"))
+        if key not in _RUNS:
+            if self.box is None:
+                self.box = self.stack.enter_context(Sandbox())
+            _RUNS[key] = self.box.small(kind)
+        return _RUNS[key]
+
 
 class SampleFailed(Exception):
     """A run sample install.py could not print."""
@@ -487,9 +572,23 @@ def sample_writers(box: Sandbox) -> dict[str, Callable[[str], str]]:
             + _fenced([_bulk_line(box)])),
         "run-check": lambda _body: _fenced(
             _lines_from(check_run(), "Checking existing files", "  Everything is in place")),
+        "check-refused": lambda _body: _fenced(_refused_lines(box.small("refused"))),
+        "from-run": lambda _body: _fenced(
+            _lines_from(box.small("from"), "Done.", count=2)),
     }
 
 
+def _refused_lines(text: str) -> list[str]:
+    """The files a check flags, each with its reason and where its cores read it."""
+    lines = text.splitlines()
+    start = next((i for i, line in enumerate(lines)
+                  if re.fullmatch(r"  \d+ files? [a-z ]+:", line)), None)
+    if start is None:
+        raise SampleFailed("the check printed no files not in place")
+    end = max(i for i, line in enumerate(lines) if line.lstrip().startswith("read by "))
+    return lines[start:end + 1]
+
+
 def render(page: str) -> str:
     """The page with each marked table and run sample written from install.py."""
     writers = {

scripts/generate_site.py

@@ -36,6 +36,7 @@ from collections import Counter
 from collections.abc import Callable, Iterator
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import TypeVar
 
 sys.path.insert(0, os.path.dirname(__file__))
 from common import (
@@ -3943,8 +3944,10 @@ def first_screen_words() -> list[str]:
     # The screen it opens first: the setup found, or the list when none
     # is. Install comes on the next one, Choose systems beside it and named
     # above on the page (tui_audit_16 P3-10).
-    return [f"It asks **{installer.TUI_FOUND_TITLE}** (or **{installer.TUI_CHOOSE_TITLE}**), "
-            f"then offers **{installer.TUI_INSTALL_ROW}**."]
+    # Nothing found, the folder comes before Install: which-pack left it
+    # out (tui_audit_17 P3-9).
+    return [f"It asks **{installer.TUI_FOUND_TITLE}** (or **{installer.TUI_CHOOSE_TITLE}** "
+            f"and **{installer.TUI_FOLDER_TITLE}**), then offers **{installer.TUI_INSTALL_ROW}**."]
 
 
 def check_pass_line() -> str:
@@ -4113,8 +4116,7 @@ def only_some_consoles(name: str, prefix: str = "") -> list[str]:
     """The way to a smaller download, in the words Get your files uses."""
     return [
         f"Only some consoles? [Pick them]({prefix}which-pack.md?platform={name}"
-        f"#only-some-consoles), or choose **{_installer().TUI_SYSTEMS_ROW}** in the "
-        "command.",
+        f"#only-some-consoles), or the command's **{_installer().TUI_SYSTEMS_ROW}**.",
         "",
     ]
 
@@ -6141,6 +6143,7 @@ def _install_box_weight(lines: list[str]) -> int:
 
 # A system card: whether it is a variant, its lines, its sha1.
 Card = tuple[bool, list[str], str]
+_Item = TypeVar("_Item")
 
 
 def _reading_entries(sha1: str, consumers: FileConsumers) -> list[dict]:
@@ -6905,10 +6908,12 @@ def _cards_markdown(cards: list[Card]) -> list[str]:
     return lines
 
 
-def _parts(cards: list[Card], budget: int, sizes: list[int] | None = None) -> list[list[Card]]:
+def _parts(cards: list[_Item], budget: int,
+           sizes: list[int] | None = None) -> list[list[_Item]]:
     """Cards in runs of at most ``budget`` bytes of Markdown, in order.
-    ``sizes`` gives each card's weight when it is not its own Markdown."""
-    parts: list[list[Card]] = [[]]
+    ``sizes`` gives each card's weight when it is not its own Markdown,
+    and is required for items other than cards."""
+    parts: list[list[_Item]] = [[]]
     size = 0
     for index, card in enumerate(cards):
         weight = sizes[index] if sizes else sum(len(line) + 1 for line in card[1])
@@ -7815,6 +7820,17 @@ def _build_rows(profile: dict) -> list[str]:
     return rows
 
 
+def _entry_label(f: dict, profile: dict) -> str:
+    """The name a file card goes by, its title without the badges."""
+    fname = f.get("name", "")
+    if f is any_name_bios(profile):
+        # Found by content: the name is an example, any image of the system works.
+        system = system_name((profile.get("systems") or [""])[0])
+        return (f"a {system} BIOS (example: {_code(fname)})" if _named_example(str(fname))
+                else f"a {system} BIOS, any file name")
+    return _code(fname)
+
+
 def _render_emulator_file(
     f: dict,
     profile: dict,
@@ -7876,12 +7892,7 @@ def _render_emulator_file(
     lines.append(markup.block_open("div", 'class="rb-file-entry"'))
     lines.append("")
     marker = f'<span id="{alias}"></span>' if alias else ""
-    label = _code(fname)
-    if f is any_name_bios(profile):
-        # Found by content: the name is an example, any image of the system works.
-        system = system_name((profile.get("systems") or [""])[0])
-        label = (f"a {system} BIOS (example: {_code(fname)})" if _named_example(str(fname))
-                 else f"a {system} BIOS, any file name")
+    label = _entry_label(f, profile)
     title = f"{marker}{label}" if badges_apart else f"{marker}{label} {badge_str}"
     lines.append(_card_title(title, anchor, weight=5))
     lines.append("")
@@ -9222,6 +9233,14 @@ EMULATOR_CARDS_PER_PAGE = 220
 # since siteentries.py, about five elements and 60 bytes more: at 220 a
 # part, MAME's first was 612 KB and 9,777 elements.
 EMULATOR_CARDS_PER_PART = 180
+# The words a part's rows may show, every card closed. A count of cards
+# held while the rows did not grow: once 888 of MAME's entries named
+# their system, each row said "for <system>" and the first part showed
+# 1,931 words for 1,180 before. The frame of a part (headings, filter
+# boxes, the links to the other parts, the footer) shows 100 to 170 more,
+# and tests/test_site_entry_tables.py holds a part under 1,693: about a
+# tenth is left as margin.
+EMULATOR_PART_WORDS = 1_350
 
 
 # The headings an emulator's cards go under, in page order: what it needs
@@ -9246,16 +9265,56 @@ def _display_order(files: list) -> list[int]:
     return sorted(range(len(files)), key=lambda i: _SECTION_RANK[_card_section(files[i])])
 
 
-def emulator_parts(name: str, files: list) -> list[tuple[str, list[int]]]:
+def _html_words(fragment: str) -> int:
+    """The words a browser shows for an HTML fragment, each tag a break."""
+    return len(html.unescape(re.sub(r"<[^>]*>", " ", fragment)).split())
+
+
+def _entry_row_words(files: list[dict], profile: dict,
+                     available: Callable[[dict], bool]) -> list[int]:
+    """The words each entry's row shows with its card closed: its title
+    and its cells, an empty cell written "-" in a column another row of
+    its section fills. A part's table drops a column its own rows leave
+    empty: counted over the whole section, a row never shows more."""
+    cells = [_entry_cells(f, profile, available) for f in files]
+    filled: dict[str, set[int]] = {}
+    for f, row in zip(files, cells):
+        filled.setdefault(_card_section(f), set()).update(
+            column for column, cell in enumerate(row) if cell)
+    return [_html_words(_entry_label(f, profile)) + sum(
+                _html_words(cell) or column in filled[_card_section(f)]
+                for column, cell in enumerate(row))
+            for f, row in zip(files, cells)]
+
+
+def emulator_parts(name: str, files: list, profile: dict | None = None,
+                   available: Callable[[dict], bool] | None = None,
+                   ) -> list[tuple[str, list[int]]]:
     """The pages an emulator's cards go to, with the entries each holds,
-    or none when they fit on the emulator's page. Balanced, in page order."""
+    or none when they fit on the emulator's page. In page order, at most
+    EMULATOR_PART_WORDS shown and EMULATOR_CARDS_PER_PART entries a part,
+    the parts balanced by words. ``available`` is the page's own test of
+    a held file, the status a row shows: without it every row is weighed
+    "in repo", the longest status."""
     if len(files) <= EMULATOR_CARDS_PER_PAGE:
         return []
+    profile = profile if profile is not None else {"files": files}
     order = _display_order(files)
-    count = -(-len(files) // EMULATOR_CARDS_PER_PART)
-    size = -(-len(files) // count)
-    return [(f"{name}-files-{index + 1}", order[start:start + size])
-            for index, start in enumerate(range(0, len(files), size))]
+    words = _entry_row_words(files, profile, available or (lambda _f: True))
+    sizes = [words[i] for i in order]
+    ceiling = max(EMULATOR_PART_WORDS, max(sizes))
+    count = max(len(_parts(order, ceiling, sizes)), -(-len(files) // EMULATOR_CARDS_PER_PART))
+    while True:
+        # The fewest words a part can hold for the parts to be ``count``:
+        # cut at the ceiling, the last part took what the others left.
+        share = min(-(-sum(sizes) // count), ceiling)
+        runs = _parts(order, share, sizes)
+        while len(runs) > count:
+            share = min(share + max(1, share // 50), ceiling)
+            runs = _parts(order, share, sizes)
+        if max(len(run) for run in runs) <= EMULATOR_CARDS_PER_PART:
+            return [(f"{name}-files-{index}", run) for index, run in enumerate(runs, 1)]
+        count += 1
 
 
 # From this many entries an emulator page lists them as one table, each
@@ -9274,26 +9333,28 @@ def _entry_rows(files: list[dict], cards: list[list[str]], profile: dict,
                 available: Callable[[dict], bool]) -> list[siteentries.Row]:
     """Each entry as a table row: its card's title and body, the number of
     members an archive holds, its badges and its hash."""
-    many = len(profile.get("systems") or []) > 1
     rows = []
     for f, card in zip(files, cards):
         title, body = siteentries.card_title_and_body(card)
-        for_system = system_name(str(f["system"])) if f.get("system") and many else ""
-        contents = f.get("contents") or []
-        badges = _file_badges(f, available(f), for_system)
-        # The first badge says whether the entry is needed: under Required,
-        # Optional or Comes with the app it repeats the section heading.
-        # One element and two words fewer on each of MAME's 1,563 rows.
-        if badges and badges[0].endswith(_SECTION_BADGE.get(_card_section(f), "\0")):
-            badges = badges[1:]
-        rows.append(siteentries.Row(title, body, [
-            f"{len(contents):,}" if contents else "",
-            " ".join(badges),
-            _hash_short(f),
-        ]))
+        rows.append(siteentries.Row(title, body, _entry_cells(f, profile, available)))
     return rows
 
 
+def _entry_cells(f: dict, profile: dict, available: Callable[[dict], bool]) -> list[str]:
+    """An entry's cells beside its title: the members an archive holds,
+    its badges and its hash."""
+    many = len(profile.get("systems") or []) > 1
+    for_system = system_name(str(f["system"])) if f.get("system") and many else ""
+    contents = f.get("contents") or []
+    badges = _file_badges(f, available(f), for_system)
+    # The first badge says whether the entry is needed: under Required,
+    # Optional or Comes with the app it repeats the section heading.
+    # One element and two words fewer on each of MAME's 1,563 rows.
+    if badges and badges[0].endswith(_SECTION_BADGE.get(_card_section(f), "\0")):
+        badges = badges[1:]
+    return [f"{len(contents):,}" if contents else "", " ".join(badges), _hash_short(f)]
+
+
 def _entries_table(files: list[dict], group: list[int], rows: list[siteentries.Row]) -> list[str]:
     """The rows of ``group``, one section's entries, as one table."""
     key = _card_section(files[group[0]])
@@ -9361,15 +9422,17 @@ def _summary_table(files: list[dict], indexes: list[int], cell: Callable[[int, s
     ]
 
 
-def plan_emulator_pages(profiles: dict) -> None:
-    """Record which page holds each emulator card, before any page links one."""
+def plan_emulator_pages(profiles: dict,
+                        available: Callable[[dict], bool] | None = None) -> None:
+    """Record which page holds each emulator card, before any page links one.
+    ``available`` is the test the pages use for a held file."""
     stems = set()
     for name, profile in sorted(profiles.items()):
         if profile.get("type") in ("alias", "test"):
             continue
         files = profile.get("files") or []
         anchors = emulator_file_anchors(files)
-        for stem, entries in emulator_parts(name, files):
+        for stem, entries in emulator_parts(name, files, profile, available):
             stems.add(stem)
             record_emulator_cards(name, stem, [anchors[i] for i in entries])
     clash = stems & set(profiles)
@@ -9725,7 +9788,7 @@ def emulator_pages(
 
         anchors = emulator_file_anchors(files)
         aliases = emulator_file_aliases(files, anchors)
-        parts = emulator_parts(name, files)
+        parts = emulator_parts(name, files, profile, _file_available)
         data_href = f"{name}.md#data-folder" if parts else "#data-folder"
         table_form = markup.starlight() and (bool(parts) or len(files) >= EMULATOR_TABLE_MIN)
         cards = [
@@ -11416,8 +11479,8 @@ def generate_which_pack() -> str:
         "",
         "## Only some consoles?",
         "",
-        "Pick them here, and get the files and the command. Or run the command "
-        f"below and choose **{_installer().TUI_SYSTEMS_ROW}**.",
+        "Pick them here, and get the files and the command. Or use the command's "
+        f"**{_installer().TUI_SYSTEMS_ROW}**.",
         "",
         "<FilePicker />",
         "",
@@ -11836,8 +11899,11 @@ def _generate(args: argparse.Namespace) -> None:
     # report the same files.
     gap_report = build_emulator_gap_report(profiles, coverages, db, suppl_names)
 
+    # The test each emulator page builds for itself: the parts a page is
+    # cut into weigh the status its rows show.
+    file_held = _availability_check(db, suppl_names)
+    plan_emulator_pages(profiles, file_held)
     # Build system_id -> manufacturer page map (needed by all generators)
-    plan_emulator_pages(profiles)
     print("Building system cross-reference map...")
     manufacturers = _group_by_manufacturer(db)
     set_system_page_map(
@@ -12003,7 +12069,7 @@ def _generate(args: argparse.Namespace) -> None:
                 or profile.get("pin_frozen")):
             SEARCH_DEMOTED.add(f"emulators/{name}.md")
         files = profile.get("files") or []
-        for stem, entries in emulator_parts(name, files):
+        for stem, entries in emulator_parts(name, files, profile, file_held):
             PAGE_DATA[f"emulators/{stem}.md"] = emulator_structured_data(
                 {**profile, "files": [files[i] for i in entries]})
             PART_PAGES.add(f"emulators/{stem}.md")

scripts/packextras.py

@@ -1377,8 +1377,17 @@ class _ClaimRanks:
 
 
 def _names_content(entry: dict) -> bool:
-    """Whether a report entry declares the bytes it wants, not only a name."""
-    return any(entry.get(field) for field in ("sha1", "md5", "sha256", "crc32"))
+    """Whether a report entry declares the bytes it wants, not only a name.
+
+    The claims folded onto the entry count: bsnes asks for SGB1.sfc by name
+    and bsnes-hd pins its md5 under the same system, and the entry carrying
+    both went after nSide's SGB1.sfc/ tree, which took the path from the file.
+    """
+    return any(
+        claim.get(field)
+        for claim in (entry, *entry.get("other_claims", []))
+        for field in ("sha1", "md5", "sha256", "crc32")
+    )
 
 
 def _save_fields(extra: dict, pack_path: str, placement: ExtraPlacement) -> None:

scripts/pipeline.py

@@ -21,6 +21,7 @@ Steps, numbered as the run prints them:
   4b   generate_pack.py --manifest  install manifests
   4c   --manifest-targets           per-target core lists
   4d   bundled_only.py --strict     copies only an emulator install provides
+  4e   generate_games_index.py      install/games/: what the games-on-card mode reads
   5    consistency check            verify == pack: counts, paths, every emulator pack
   6    --verify-packs               pack integrity, extraction and hashes
   7    generate_readme.py           README.md and CONTRIBUTING.md
@@ -125,6 +126,7 @@ STEP_LABELS: dict[str, str] = {
     "generate_manifests": "4b generate install manifests",
     "generate_target_manifests": "4c generate target manifests",
     "bundled_only": "4d copies only an emulator install provides",
+    "games_index": "4e games index",
     "consistency": "5 consistency check",
     "pack_integrity": "6 pack integrity",
     "generate_readme": "7 generate readme",
@@ -1040,6 +1042,18 @@ def main() -> None:
     results["bundled_only"] = ok
     all_ok &= ok
 
+    # Step 4e: What the installer reads for the games on a card: rom folders,
+    # game catalogs, header and region rules, arcade parents, and the
+    # fallback sets, which read the install lists. Runs under --skip-packs
+    # too, on the committed lists, so a rescrape of the rom folders or a DAT
+    # import reaches install/games/ in the same run.
+    ok, _ = run(
+        [sys.executable, "scripts/generate_games_index.py", "--quiet"],
+        STEP_LABELS["games_index"],
+    )
+    results["games_index"] = ok
+    all_ok &= ok
+
     # Step 5: Consistency check. verify describes the full pack, and a run that
     # built only a reduced one (--source platform or truth) has nothing to
     # compare it with, and says so instead of failing on an empty parse.

scripts/romfolders.py

@@ -0,0 +1,194 @@
+"""Rom folders of each setup: where its games live and what each folder holds.
+
+`platforms/romfolders/<setup>.yml` records what a frontend's own source says
+about its game folders, read at a pinned revision: the roms root relative to
+the BIOS folder, each folder with the system name the OS gives it, the
+extensions it lists and the core it starts by default. Those files hold the
+OS's facts only. Which repository system a folder is, and which profile its
+core is, is decided here, through the one alias table in common.py
+(`SYSTEM_ALIASES`, `_norm_system_id`) and the profiles' `cores:` lists, so a
+change to either reaches every setup without a rescrape.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from common import (
+    _norm_system_id,
+    list_registered_platforms,
+    load_emulator_profiles,
+    load_platform_config,
+    native_layer,
+    preferred_profile,
+    upstream_profile_index,
+)
+from safeparse import yaml_load
+
+ROMFOLDERS_DIR = Path("platforms") / "romfolders"
+SOURCES_DIR = ROMFOLDERS_DIR / "_sources"
+
+# "<source key> <path>:<line>[-<line>]", several joined by "; ".
+_REF_PART = re.compile(r"^(?P<key>[a-z0-9][a-z0-9-]*) (?P<path>\S+?)(?::(?P<lines>\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*))?$")
+
+
+def split_refs(source_ref: str) -> list[tuple[str, str, str | None]]:
+    """Each (source key, path, lines) a source_ref names, or ValueError."""
+    parts: list[tuple[str, str, str | None]] = []
+    for raw in source_ref.split("; "):
+        match = _REF_PART.match(raw.strip())
+        if not match:
+            raise ValueError(f"source_ref part {raw!r} is not '<source> <path>:<line>'")
+        parts.append((match["key"], match["path"], match["lines"]))
+    return parts
+
+
+def setup_paths(directory: Path = ROMFOLDERS_DIR) -> list[Path]:
+    """The setup files of a directory, without the sources file."""
+    return sorted(p for p in Path(directory).glob("*.yml") if not p.name.startswith("_"))
+
+
+def load_setup(path: Path) -> dict:
+    with Path(path).open(encoding="utf-8") as handle:
+        return yaml_load(handle)
+
+
+def load_setups(directory: Path = ROMFOLDERS_DIR) -> dict[str, dict]:
+    """Every setup document of a directory, keyed by its `setup:` id."""
+    return {doc["setup"]: doc for doc in (load_setup(p) for p in setup_paths(directory))}
+
+
+@dataclass
+class Resolution:
+    """What a folder resolved to, and how."""
+
+    systems: list[str]
+    basis: str
+
+
+@dataclass
+class SystemResolver:
+    """Folder and system names to repository system ids.
+
+    The universe is every system a profile or a platform file names. A name
+    resolves, in this order, as one of those ids, as a name a platform's own
+    file gives a system (`native_system`, the folder Batocera calls `psx`),
+    or through `_norm_system_id`, which applies SYSTEM_ALIASES then drops the
+    maker prefix and separators. A name nothing matches stays unresolved:
+    a new abbreviation goes into SYSTEM_ALIASES, never into a table here.
+
+    Ids that normalize alike are one machine spelled several ways
+    (`sega-megadrive`, `sega-mega-drive`, `mega-drive`). Each class answers
+    with one canonical id, the spelling most profiles use, and
+    `equivalents()` gives the whole class back to whoever matches a
+    profile's `systems:` or a manifest's `systems`.
+    """
+
+    ids: set[str]
+    profile_use: dict[str, int]
+    native: dict[str, dict[str, set[str]]]
+    by_norm: dict[str, set[str]] = field(default_factory=dict)
+    canonical_of: dict[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        for sid in self.ids:
+            self.by_norm.setdefault(_norm_system_id(sid), set()).add(sid)
+        for members in self.by_norm.values():
+            best = min(members, key=lambda s: (-self.profile_use.get(s, 0), -len(s), s))
+            for sid in members:
+                self.canonical_of[sid] = best
+
+    @classmethod
+    def from_repo(
+        cls, emulators_dir: str = "emulators", platforms_dir: str = "platforms"
+    ) -> SystemResolver:
+        profiles = load_emulator_profiles(emulators_dir)
+        use: dict[str, int] = {}
+        for profile in profiles.values():
+            for sid in profile.get("systems") or []:
+                use[str(sid)] = use.get(str(sid), 0) + 1
+        ids = set(use)
+        native: dict[str, dict[str, set[str]]] = {}
+        for platform in list_registered_platforms(platforms_dir, include_archived=True):
+            config = load_platform_config(platform, platforms_dir)
+            ids.update(config.get("systems", {}))
+            names: dict[str, set[str]] = {}
+            for sys_id, system in (native_layer(config) or {}).get("systems", {}).items():
+                for entry in system.get("files", []):
+                    name = entry.get("native_system")
+                    if isinstance(name, str) and name.strip():
+                        names.setdefault(name.strip().lower(), set()).add(sys_id)
+            native[platform] = names
+        return cls(ids=ids, profile_use=use, native=native)
+
+    def canonical(self, sid: str) -> str:
+        return self.canonical_of.get(sid, sid)
+
+    def equivalents(self, sid: str) -> set[str]:
+        """Every id spelling the same machine as sid, sid included."""
+        return set(self.by_norm.get(_norm_system_id(sid), ())) | {sid}
+
+    def _answer(self, found: set[str], basis: str) -> Resolution:
+        return Resolution(sorted({self.canonical(s) for s in found}), basis)
+
+    def resolve(self, names: list[str], platform: str = "") -> Resolution:
+        """The systems the first name that resolves designates."""
+        for name in names:
+            if not name:
+                continue
+            lowered = name.strip().lower()
+            if lowered in self.ids:
+                return self._answer({lowered}, "id")
+            own = self.native.get(platform, {}).get(lowered)
+            if own and len(own) == 1:
+                return self._answer(own, f"native name of {platform}")
+            norm = self.by_norm.get(_norm_system_id(lowered))
+            if norm:
+                return self._answer(norm, "normalized id")
+            claimed = [
+                (plat, owners) for plat, table in sorted(self.native.items())
+                if (owners := table.get(lowered)) and len(owners) == 1
+            ]
+            if claimed:
+                plat, owners = claimed[0]
+                return self._answer(owners, f"native name of {plat}")
+        return Resolution([], "unresolved")
+
+
+@dataclass
+class ProfileResolver:
+    """An OS's core name to the repository profile that documents it."""
+
+    index: dict[str, set[str]]
+
+    @classmethod
+    def from_repo(cls, emulators_dir: str = "emulators") -> ProfileResolver:
+        return cls(upstream_profile_index(load_emulator_profiles(emulators_dir)))
+
+    def resolve(self, core: str | None) -> str | None:
+        if not core:
+            return None
+        # RetroPie names a libretro package lr-<core> (scriptmodules/libretrocores).
+        bare = core.removeprefix("lr-")
+        spellings = (core, core.lower(), bare, bare.replace("-", "_"), bare.lower().replace("-", "_"))
+        for name in dict.fromkeys(spellings):
+            found = preferred_profile(self.index, name)
+            if found:
+                return found
+        return None
+
+
+_TAG = re.compile(r"\(([^()]+)\)\s*$")
+
+
+def folder_names(row: dict) -> list[str]:
+    """The names a folder row is known by, the OS's system name first.
+
+    A folder named for people and tagged for the frontend, `Sony PlayStation
+    (PS)` on CrossMix-OS, is also known by its tag.
+    """
+    names = [row.get("name") or "", row["folder"], *(row.get("aliases") or [])]
+    names += [m.group(1).strip() for n in list(names) if (m := _TAG.search(n))]
+    return [n for n in dict.fromkeys(names) if n]

scripts/scraper/game_dat_importer.py

@@ -0,0 +1,514 @@
+"""Import game catalogs from No-Intro and Redump DATs, one file per system.
+
+The games-on-card mode identifies a game from its bytes: a cartridge by the
+crc32 and size a ZIP central directory gives without inflating, a disc by
+the first data track or the serial its header carries. These catalogs hold
+those keys with the title and the territory the DAT names the game for.
+They are game catalogues, not dump catalogues of BIOS files, so they live in
+games/catalogs/ and never in provenance/.
+
+    python -m scripts.scraper.game_dat_importer --fetch
+    python -m scripts.scraper.game_dat_importer --fetch --source redump
+    python -m scripts.scraper.game_dat_importer --offline --floor
+    python -m scripts.scraper.game_dat_importer --source no-intro --pack no-intro.zip
+
+No-Intro comes from the auto-datfile-generator release asset (Dat-o-Matic
+bans automated clients), Redump from redump.info's per-system datfile route.
+Every download is cached under .cache/game-dats/ with what the server said
+about it, so a second run reads nothing from the network and writes the same
+bytes. A DAT's region tag is a market label, never the value a core computes.
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import io
+import json
+import re
+import sys
+import urllib.request
+import zipfile
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+_SCRIPTS_DIR = str(Path(__file__).resolve().parents[1])
+if _SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, _SCRIPTS_DIR)
+
+from artifacts import copy_stream_atomic, write_text_atomic  # noqa: E402
+from common import parse_untrusted_xml  # noqa: E402
+from region import REGIONS, WORLD  # noqa: E402
+from region_audit import NOINTRO  # noqa: E402
+
+from .base_scraper import github_headers  # noqa: E402
+
+SOURCES = ("no-intro", "redump")
+CACHE_DIR = Path(".cache") / "game-dats"
+OUTPUT_DIR = Path("games") / "catalogs"
+
+NOINTRO_RELEASE_API = (
+    "https://api.github.com/repos/hugo19941994/auto-datfile-generator/releases/tags/Daily_Rebuild"
+)
+NOINTRO_ASSET = "no-intro.zip"
+REDUMP_DATFILE_URL = "https://redump.info/datfile/{code}"
+
+# DAT header name -> (repository system id, floor). The floor is the least
+# number of games the DAT may hold before an import is refused as truncated:
+# 95 % of the count read on 2026-11-03, rounded down.
+NOINTRO_DATS: dict[str, tuple[str, int]] = {
+    "Nintendo - Super Nintendo Entertainment System": ("nintendo-snes", 3920),
+    "Nintendo - Game Boy": ("nintendo-gb", 1897),
+    "Nintendo - Game Boy Color": ("nintendo-gbc", 1935),
+    "Nintendo - Game Boy Advance": ("nintendo-gba", 3356),
+    "Nintendo - Nintendo 64 (BigEndian)": ("nintendo-64", 1098),
+    "Sega - Mega Drive - Genesis": ("sega-megadrive", 2718),
+    "Nintendo - Nintendo Entertainment System (Headered)": ("nintendo-nes", 4276),
+    "Nintendo - Nintendo DS (Decrypted)": ("nintendo-ds", 7289),
+    "NEC - PC Engine - TurboGrafx-16": ("nec-pc-engine", 438),
+    "Sega - Master System - Mark III": ("sega-mastersystem", 665),
+    "Sega - Game Gear": ("sega-gamegear", 786),
+    "Atari - Atari Lynx (LNX)": ("atari-lynx", 17),
+    "Nintendo - Nintendo 64DD": ("nintendo-64dd", 33),
+    "Nintendo - Family Computer Disk System (FDS)": ("nintendo-fds", 384),
+    "Nintendo - Pokemon Mini": ("nintendo-pokemon-mini", 41),
+    "Atari - Atari 7800 (A78)": ("atari-7800", 7),
+    "Atari - Atari 5200": ("atari-5200", 197),
+    "Mattel - Intellivision": ("intellivision", 197),
+    "Coleco - ColecoVision": ("colecovision", 190),
+    "Magnavox - Odyssey 2": ("odyssey2", 126),
+    "GCE - Vectrex": ("vectrex", 45),
+    "Nintendo - Satellaview": ("nintendo-satellaview", 570),
+    "Nintendo - Sufami Turbo": ("nintendo-sufami-turbo", 12),
+    "NEC - PC Engine SuperGrafx": ("nec-supergrafx", 4),
+    "Sega - 32X": ("sega-32x", 202),
+    "Bandai - WonderSwan": ("wonderswan", 244),
+    "Bandai - WonderSwan Color": ("wonderswan-color", 229),
+    "SNK - NeoGeo Pocket": ("snk-neogeo-pocket", 12),
+    "SNK - NeoGeo Pocket Color": ("neo-geo-pocket-color", 121),
+}
+
+# redump.info datfile code -> (DAT header name, repository system id, floor).
+REDUMP_DATS: dict[str, tuple[str, str, int]] = {
+    "PSX": ("Sony - PlayStation", "sony-playstation", 10432),
+    "PS2": ("Sony - PlayStation 2", "sony-playstation-2", 11252),
+    "SS": ("Sega - Saturn", "sega-saturn", 2340),
+    "MCD": ("Sega - Mega CD & Sega CD", "sega-megacd", 521),
+    "DC": ("Sega - Dreamcast", "sega-dreamcast", 1442),
+    "GC": ("Nintendo - GameCube", "nintendo-gamecube", 1920),
+    "PCE": ("NEC - PC Engine CD & TurboGrafx CD", "nec-pc-engine-cd", 523),
+    "NGCD": ("SNK - Neo Geo CD", "snk-neogeo-cd", 107),
+    "3DO": ("3DO Interactive Multiplayer", "panasonic-3do", 635),
+}
+
+TERMS_READ = "2026-11-03"
+LICENCES: dict[str, dict] = {
+    "no-intro": {
+        "terms_url": "https://datomatic.no-intro.org/terms.html",
+        "terms_title": "Data Usage License",
+        "terms_updated": "2026-09-11",
+        "read": TERMS_READ,
+        "quote": (
+            'Unless otherwise stated on the website, the data made available through '
+            'this website ("Data") may be freely used, copied, reproduced, modified, '
+            "adapted, combined, published, distributed, and otherwise reused by anyone "
+            "for any lawful purpose, to the extent that the website operator owns or "
+            "controls the applicable rights or is authorized to grant such permission. "
+            "No permission, attribution, citation, backlink, or acknowledgment of the "
+            "website or its operators is required for such use."
+        ),
+        "mirror": {
+            "repo": "https://github.com/hugo19941994/auto-datfile-generator",
+            "licence": "MIT",
+            "scope": "the mirror's scripts only, the DATs it republishes stay under the terms above",
+        },
+    },
+    "redump": {
+        "terms_url": "https://redump.info/about",
+        "read": TERMS_READ,
+        "quote": (
+            "Dumpers only submit metadata and Redump only stores metadata. This "
+            "information is considered public domain to be used however people see fit"
+        ),
+    },
+}
+
+# Territory words the DATs use that scripts/region.py has no member slug
+# for. Each falls under the one super-region that contains it.
+EXTRA_TERRITORIES = {
+    "Scandinavia": "europe",
+    "Belgium": "europe",
+    "Switzerland": "europe",
+    "Austria": "europe",
+    "Ireland": "europe",
+    "United Kingdom": "uk",
+}
+_TERRITORY = {**{k: v for k, v in NOINTRO.items()}, **EXTRA_TERRITORIES}
+# A territory word with no slug: the group is a territory tag, but the
+# catalog cannot name it and records no region.
+_UNMAPPED_TERRITORIES = {"South Africa"}
+
+SERIAL_NOTE = (
+    "Serials are written as the DAT writes them, and the DAT uses both forms: "
+    "'SLES-01971' and 'SLPS 02251' in the PlayStation DAT of the same day. A "
+    "core reads the serial from the disc in its own spelling (SYSTEM.CNF holds "
+    "'SLPS_022.51', DuckStation reports 'SLPS-02251'). No source fixes a rule "
+    "joining the two, so the comparison rule is the reader's to state."
+)
+
+_PAREN = re.compile(r"\(([^)]+)\)")
+_NOINTRO_MEMBER = re.compile(r"^(?P<name>.+) \((?P<version>\d{8}-\d{6})\)\.dat$")
+_DISPOSITION = re.compile(r'filename="(?P<file>[^"]+)"')
+_REDUMP_FILE = re.compile(
+    r"^(?P<name>.+) - Datfile \((?P<count>\d+)\) \((?P<date>\d{4}-\d{2}-\d{2}) (?P<time>[\d-]+)\)\.(?:zip|dat)$"
+)
+
+
+def name_regions(name: str) -> list[str] | None:
+    """Region slugs of the first parenthesised group made only of territory
+    words, or None when the name carries no such group or a word the
+    vocabulary cannot name."""
+    for group in _PAREN.findall(name):
+        words = [w.strip() for w in group.split(",")]
+        if not words or not all(w in _TERRITORY or w in _UNMAPPED_TERRITORIES for w in words):
+            continue
+        if any(w in _UNMAPPED_TERRITORIES for w in words):
+            return None
+        slugs = sorted({_TERRITORY[w] for w in words})
+        if WORLD in slugs:
+            return [WORLD]
+        unknown = set(slugs) - REGIONS
+        if unknown:
+            raise ValueError(f"region slugs {sorted(unknown)} are outside scripts/region.py")
+        return slugs
+    return None
+
+
+@dataclass
+class DatGame:
+    name: str
+    serials: list[str]
+    roms: list[tuple[str, int, str, str]]  # (name, size, crc32, sha1)
+
+
+def parse_dat(content: bytes) -> tuple[str, str, list[DatGame]]:
+    """Header name, version and games of a Logiqx DAT, serials included."""
+    root = parse_untrusted_xml(content, "DAT files")
+    header = root.find("header")
+    name = (header.findtext("name") or "").strip() if header is not None else ""
+    version = (header.findtext("version") or "").strip() if header is not None else ""
+    games: list[DatGame] = []
+    for game in root.iter("game"):
+        serial = game.findtext("serial") or ""
+        serials = [s.strip() for s in serial.split(",") if s.strip()]
+        roms = []
+        for rom in game.findall("rom"):
+            try:
+                size = int(rom.get("size") or 0)
+            except ValueError:
+                size = 0
+            roms.append((rom.get("name") or "", size,
+                         (rom.get("crc") or "").lower(), (rom.get("sha1") or "").lower()))
+        games.append(DatGame(game.get("name") or "", serials, roms))
+    return name, version, games
+
+
+def _first_data_track(roms: list[tuple[str, int, str, str]]) -> tuple[str, int, str, str] | None:
+    for rom in roms:
+        if rom[0].lower().endswith((".cue", ".gdi")) or not rom[2]:
+            continue
+        return rom
+    return None
+
+
+def build_catalog(source: str, system: str, dat_name: str, version: str,
+                  games: list[DatGame], origin: dict) -> dict:
+    """The catalog document of one DAT."""
+    region_ids: dict[tuple[str, ...], int] = {}
+    rows: list[list] = []
+    for game in games:
+        regions = name_regions(game.name)
+        index = None
+        if regions is not None:
+            index = region_ids.setdefault(tuple(regions), len(region_ids))
+        if source == "redump":
+            track = _first_data_track(game.roms)
+            if track is None:
+                continue
+            rows.append([track[2], track[1], track[3], index, game.name, sorted(game.serials)])
+        else:
+            for _, size, crc, sha1 in game.roms:
+                if crc:
+                    rows.append([crc, size, sha1, index, game.name])
+    # Region ids are renumbered in sorted order so the table does not depend
+    # on the DAT's game order.
+    order = sorted(region_ids, key=lambda r: (len(r), r))
+    renumber = {region_ids[r]: i for i, r in enumerate(order)}
+    for row in rows:
+        if row[3] is not None:
+            row[3] = renumber[row[3]]
+    rows.sort(key=lambda r: (r[0], r[1], r[2], r[4]))
+    fields = ["crc32", "size", "sha1", "region", "title"]
+    doc = {
+        "source": source,
+        "system": system,
+        "dat": dat_name,
+        "version": version,
+        **origin,
+        "licence": LICENCES[source],
+        "fields": fields + (["serials"] if source == "redump" else []),
+        "regions": [list(r) for r in order],
+        "games": rows,
+    }
+    if source == "redump":
+        doc["serial_note"] = SERIAL_NOTE
+    return doc
+
+
+def dump_catalog(doc: dict) -> str:
+    """Compact JSON, one game per line so a refresh diffs by game."""
+    head = {k: v for k, v in doc.items() if k != "games"}
+    text = json.dumps(head, ensure_ascii=False, separators=(",", ":"))[:-1]
+    games = ",\n".join(json.dumps(g, ensure_ascii=False, separators=(",", ":")) for g in doc["games"])
+    return f'{text},"games":[\n{games}\n]}}\n'
+
+
+def write_if_changed(path: Path, text: str) -> bool:
+    if path.is_file() and path.read_text(encoding="utf-8") == text:
+        return False
+    path.parent.mkdir(parents=True, exist_ok=True)
+    write_text_atomic(str(path), text)
+    return True
+
+
+# Fetching ---------------------------------------------------------------
+
+
+class NotCachedError(RuntimeError):
+    """An offline run asked for a download the cache does not hold."""
+
+
+def _download(url: str, destination: Path, headers: dict[str, str] | None = None) -> dict[str, str]:
+    """Stream url to destination and return the response headers kept."""
+    destination.parent.mkdir(parents=True, exist_ok=True)
+    request = urllib.request.Request(url, headers={"User-Agent": "retrobios", **(headers or {})})
+    with urllib.request.urlopen(request, timeout=300) as response:
+        copy_stream_atomic(response, str(destination))
+        return {"content_disposition": response.headers.get("Content-Disposition", "")}
+
+
+def _today() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%d")
+
+
+def fetch_nointro(cache_dir: Path, offline: bool, refresh: bool) -> tuple[Path, dict]:
+    """The No-Intro pack and what the release said about it."""
+    meta_path = cache_dir / "no-intro.json"
+    if meta_path.is_file() and not refresh:
+        meta = json.loads(meta_path.read_text(encoding="utf-8"))
+        pack = cache_dir / meta["file"]
+        if pack.is_file():
+            return pack, meta
+    if offline:
+        raise NotCachedError(f"{meta_path} or its pack is not cached")
+    request = urllib.request.Request(NOINTRO_RELEASE_API, headers=github_headers())
+    with urllib.request.urlopen(request, timeout=60) as response:
+        release = json.loads(response.read())
+    asset = next(a for a in release["assets"] if a["name"] == NOINTRO_ASSET)
+    stamp = asset["updated_at"].replace(":", "").replace("-", "")
+    file = f"no-intro-{stamp}.zip"
+    pack = cache_dir / file
+    if not pack.is_file():
+        _download(asset["browser_download_url"], pack)
+    if pack.stat().st_size != asset["size"]:
+        pack.unlink()
+        raise RuntimeError(f"{NOINTRO_ASSET}: size differs from the release's {asset['size']}")
+    meta = {
+        "file": file,
+        "url": asset["browser_download_url"],
+        "asset_updated_at": asset["updated_at"],
+        "asset_size": asset["size"],
+        "fetched": _today(),
+    }
+    write_text_atomic(str(meta_path), json.dumps(meta, indent=1) + "\n")
+    return pack, meta
+
+
+def fetch_redump(code: str, cache_dir: Path, offline: bool, refresh: bool) -> tuple[Path, dict]:
+    """One redump.info datfile and the name its server gave it."""
+    folder = cache_dir / "redump"
+    meta_path = folder / f"{code}.json"
+    pack = folder / f"{code}.zip"
+    if meta_path.is_file() and pack.is_file() and not refresh:
+        return pack, json.loads(meta_path.read_text(encoding="utf-8"))
+    if offline:
+        raise NotCachedError(f"{pack} is not cached")
+    url = REDUMP_DATFILE_URL.format(code=code)
+    kept = _download(url, pack)
+    match = _DISPOSITION.search(kept["content_disposition"])
+    if not match:
+        raise RuntimeError(f"{url} answered without a datfile name")
+    meta = {"file": match["file"], "url": url, "fetched": _today()}
+    write_text_atomic(str(meta_path), json.dumps(meta, indent=1) + "\n")
+    return pack, meta
+
+
+# Reading packs ----------------------------------------------------------
+
+
+def _iter_dats(pack: Path) -> Iterator[tuple[str, bytes]]:
+    """(member name, bytes) of every .dat in a ZIP, a directory or a file."""
+    if pack.is_dir():
+        for path in sorted(pack.rglob("*.dat")):
+            yield path.name, path.read_bytes()
+        return
+    if zipfile.is_zipfile(pack):
+        with zipfile.ZipFile(pack) as archive:
+            for info in sorted(archive.infolist(), key=lambda i: i.filename):
+                if info.filename.lower().endswith(".dat"):
+                    yield Path(info.filename).name, archive.read(info)
+        return
+    yield pack.name, pack.read_bytes()
+
+
+def import_nointro(pack: Path, meta: dict) -> list[dict]:
+    docs = []
+    for member, data in _iter_dats(pack):
+        match = _NOINTRO_MEMBER.match(member)
+        name = match["name"] if match else None
+        if name not in NOINTRO_DATS:
+            # A member named without the version stamp is read for its header.
+            if match:
+                continue
+            name = parse_dat(data)[0]
+            if name not in NOINTRO_DATS:
+                continue
+        dat_name, version, games = parse_dat(data)
+        origin = {
+            "fetched_from": meta.get("url", ""),
+            "fetched": meta.get("fetched", ""),
+            "archive": {"member": member, "asset_updated_at": meta.get("asset_updated_at", "")},
+        }
+        docs.append(build_catalog("no-intro", NOINTRO_DATS[dat_name][0], dat_name, version,
+                                  games, origin))
+    return docs
+
+
+def import_redump(pack: Path, meta: dict) -> list[dict]:
+    docs = []
+    by_name = {name: (code, system) for code, (name, system, _) in REDUMP_DATS.items()}
+    for member, data in _iter_dats(pack):
+        dat_name, version, games = parse_dat(data)
+        if dat_name not in by_name:
+            continue
+        file = meta.get("file", member)
+        stamp = _REDUMP_FILE.match(file)
+        origin = {
+            "fetched_from": meta.get("url", ""),
+            "fetched": meta.get("fetched", ""),
+            "archive": {"member": member, "file": file,
+                        "dat_date": stamp["date"] if stamp else ""},
+        }
+        docs.append(build_catalog("redump", by_name[dat_name][1], dat_name, version,
+                                  games, origin))
+    return docs
+
+
+def coverage(doc: dict) -> dict:
+    games = {row[4] for row in doc["games"]}
+    out = {
+        "system": doc["system"],
+        "games": len(games),
+        "rows": len(doc["games"]),
+        "with_region": sum(1 for row in doc["games"] if row[3] is not None),
+    }
+    if doc["source"] == "redump":
+        by_serial: dict[str, set] = {}
+        for row in doc["games"]:
+            for serial in row[5]:
+                by_serial.setdefault(serial, set()).add(row[3])
+        out["serials"] = len(by_serial)
+        out["multi_region_serials"] = sum(1 for v in by_serial.values() if len(v) > 1)
+    return out
+
+
+def floor_of(doc: dict) -> int:
+    if doc["source"] == "no-intro":
+        return NOINTRO_DATS[doc["dat"]][1]
+    return next(f for name, _, f in REDUMP_DATS.values() if name == doc["dat"])
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
+    parser.add_argument("--source", choices=SOURCES, action="append",
+                        help="limit to one source (repeatable, default both)")
+    mode = parser.add_mutually_exclusive_group()
+    mode.add_argument("--fetch", action="store_true", help="download what the cache lacks")
+    mode.add_argument("--offline", action="store_true", help="read the cache only (default)")
+    mode.add_argument("--pack", type=Path, help="import a local ZIP, directory or DAT")
+    parser.add_argument("--refresh", action="store_true", help="ask the servers again")
+    parser.add_argument("--floor", action="store_true",
+                        help="fail when a system holds fewer games than its stored floor")
+    parser.add_argument("--cache-dir", type=Path, default=CACHE_DIR,
+                        help=f"downloaded packs (default {CACHE_DIR})")
+    parser.add_argument("--output-dir", type=Path, default=OUTPUT_DIR,
+                        help=f"where the catalogs are written (default {OUTPUT_DIR})")
+    parser.add_argument("--dry-run", action="store_true", help="print coverage, write nothing")
+    args = parser.parse_args()
+
+    sources = args.source or list(SOURCES)
+    if args.pack and len(sources) != 1:
+        parser.error("--pack needs one --source")
+    offline = not args.fetch
+    docs: list[dict] = []
+    try:
+        for source in sources:
+            if args.pack:
+                meta = {"url": "", "fetched": ""}
+                docs += (import_nointro if source == "no-intro" else import_redump)(args.pack, meta)
+            elif source == "no-intro":
+                pack, meta = fetch_nointro(args.cache_dir, offline, args.refresh)
+                docs += import_nointro(pack, meta)
+            else:
+                for code in REDUMP_DATS:
+                    pack, meta = fetch_redump(code, args.cache_dir, offline, args.refresh)
+                    docs += import_redump(pack, meta)
+    except (NotCachedError, OSError, RuntimeError, ValueError) as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        if isinstance(exc, NotCachedError):
+            print("Run with --fetch to download it.", file=sys.stderr)
+        return 1
+
+    expected = {(s, sid) for s in sources if not args.pack
+                for sid in ([v[0] for v in NOINTRO_DATS.values()] if s == "no-intro"
+                            else [v[1] for v in REDUMP_DATS.values()])}
+    missing = expected - {(d["source"], d["system"]) for d in docs}
+    failures = [f"{s}/{sid}: DAT not in the pack" for s, sid in sorted(missing)]
+    raw_total = gz_total = 0
+    print(f"{'source':9} {'system':26} {'games':>6} {'rows':>6} {'region':>6} "
+          f"{'serials':>7} {'multi':>5} {'raw':>9} {'gzip':>8}")
+    for doc in sorted(docs, key=lambda d: (d["source"], d["system"])):
+        text = dump_catalog(doc)
+        raw = len(text.encode())
+        gz = len(gzip.compress(text.encode(), compresslevel=9, mtime=0))
+        raw_total += raw
+        gz_total += gz
+        cov = coverage(doc)
+        print(f"{doc['source']:9} {doc['system']:26} {cov['games']:>6} {cov['rows']:>6} "
+              f"{cov['with_region']:>6} {cov.get('serials', '-'):>7} "
+              f"{cov.get('multi_region_serials', '-'):>5} {raw:>9} {gz:>8}")
+        if args.floor and cov["games"] < floor_of(doc):
+            failures.append(f"{doc['source']}/{doc['system']}: {cov['games']} games, "
+                            f"floor {floor_of(doc)}")
+        if not args.dry_run:
+            write_if_changed(args.output_dir / doc["source"] / f"{doc['system']}.json", text)
+    print(f"{len(docs)} catalogs, {raw_total} bytes raw, {gz_total} bytes gzip")
+    for failure in failures:
+        print(f"FAIL {failure}", file=sys.stderr)
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/scraper/mame_devices.py

@@ -272,6 +272,33 @@ def attribute_systems(
     return {name: sorted(found) for name, found in systems.items()}
 
 
+def device_systems(
+    machines: dict[str, MachineLinks],
+    seeds: dict[str, str],
+    fallback: str,
+) -> dict[str, list[str]]:
+    """Systems whose machines can load each set, unplaced machines under *fallback*.
+
+    attribute_systems files a set under the systems of the machines a seed
+    places. A machine no seed places (a computer no platform runs through
+    this profile, a game outside every BIOS family) still loads its devices,
+    and MAME starts it from its set name, the way an arcade folder passes a
+    game: it counts as a machine of *fallback*. A set only such machines
+    load (an Amiga keyboard, an ISA card) then goes under *fallback* alone,
+    and a set both kinds load (ym2413, for the CoCo and for arcade games)
+    under both. A set no runnable machine of the release instantiates keeps
+    no system: the document never names its machine.
+    """
+    if not fallback:
+        return attribute_systems(machines, seeds)
+    unplaced = {
+        name: fallback
+        for name, links in machines.items()
+        if not links.isdevice and not _seed_system(name, machines, seeds)
+    }
+    return attribute_systems(machines, {**seeds, **unplaced})
+
+
 def platform_seeds(
     stated: Iterable[tuple[str, str]],
     machines: dict[str, MachineLinks],

scripts/scraper/mame_hash_scraper.py

@@ -48,6 +48,7 @@ from .mame_devices import (
     attribute_softlists,
     attribute_systems,
     default_floppy_samples,
+    device_systems,
     drive_form_factor,
     floppy_drives,
     floppy_sample_systems,
@@ -70,6 +71,9 @@ _CLONE_DIR = _ROOT / "tmp" / f"mame-{os.getpid()}"
 _EMULATORS_DIR = _ROOT / "emulators"
 _PLATFORMS_DIR = _ROOT / "platforms"
 _FLOPPY_SOURCE = "src/devices/imagedev/floppy.cpp"
+# Where a device set goes when only machines no seed places load it: MAME
+# starts such a machine from its set name, as an arcade folder passes one.
+UNPLACED_SYSTEM = "arcade"
 _REPO_URL = "https://github.com/mamedev/mame.git"
 _STALE_HOURS = 24
 
@@ -570,9 +574,14 @@ def _merge_devices(args: argparse.Namespace) -> None:
     stated, conflicts = platform_seeds(
         _platform_archives(), machines, profile.get("systems") or [], written
     )
-    systems = attribute_systems(machines, {**stated, **written})
+    seeds = {**stated, **written}
+    systems = attribute_systems(machines, seeds)
+    listed = profile.get("systems") or []
+    filed = device_systems(
+        machines, seeds, UNPLACED_SYSTEM if UNPLACED_SYSTEM in listed else ""
+    )
     for set_name, device in devices.items():
-        device["systems"] = systems.get(set_name, [])
+        device["systems"] = filed.get(set_name, [])
     report = merge_mame_devices(str(profile_path), devices, write=not args.dry_run)
 
     shipped = upstream.list_directory(repo, commit, "hash", upstream.CACHE_DIR)
@@ -602,10 +611,12 @@ def _merge_devices(args: argparse.Namespace) -> None:
     for set_name in report["conflict"]:
         print(f"    ! {set_name}.zip is listed under another category, left as written")
     by_count = Counter(len(device["systems"]) for device in devices.values())
+    fallen = sum(1 for device in devices.values() if UNPLACED_SYSTEM in device["systems"])
     print(
         f"  systems: {by_count[1]} sets under one system, "
         f"{sum(n for k, n in by_count.items() if k > 1)} under several, "
-        f"{by_count[0]} loaded by no placed machine "
+        f"{fallen} of them under {UNPLACED_SYSTEM} (loaded by machines no entry places), "
+        f"{by_count[0]} instantiated by no machine of the release "
         "(they serve every system of the profile)"
     )
     print(

scripts/scraper/redump_dat_scraper.py

@@ -1,9 +1,11 @@
 """Fetch Redump BIOS DAT files and write a provenance snapshot.
 
-Redump serves BIOS DATs as static clrmamepro files linked from
-https://redump.info/downloads under /static/bios/. The downloads page
-is scanned for those links, so new BIOS DATs are picked up without
-code changes.
+Redump serves BIOS DATs as Logiqx files linked from
+https://redump.info/downloads. The links were static files under
+/static/bios/<name>.dat and are now one route per system,
+/bios/<CODE> (PSX, PS2, GC, XBOX), served as an attachment. The
+downloads page is scanned for either shape, so new BIOS DATs are
+picked up without code changes.
 
 Run manually, then commit the snapshot:
     python -m scripts.scraper.redump_dat_scraper --dry-run
@@ -40,6 +42,7 @@ def _fetch(url: str) -> str:
 def discover_bios_datfiles(downloads_html: str) -> list[str]:
     """Extract BIOS DAT paths from the downloads page."""
     paths = set(re.findall(r'href="(/static/bios/[^"]+\.dat)"', downloads_html))
+    paths.update(re.findall(r'href="(/bios/[A-Za-z0-9_-]+)"', downloads_html))
     return sorted(paths)
 
 

scripts/scraper/romfolders/__init__.py

@@ -0,0 +1,193 @@
+"""Scrape each setup's rom folders from its own source, at a pinned revision.
+
+`platforms/romfolders/_sources/<setup>.yml` names the repositories and
+the commits to read, the format of the system list, and the facts a parser
+cannot read from a table (roms root, case rule, folders the OS never scans),
+each with its `source_ref`. A format parser reads the system list and returns
+one row per folder. The setup file it writes carries the OS's own names only:
+`scripts/romfolders.py` resolves them to repository systems and profiles.
+
+Every fetch goes through `upstream.fetch_file`, cached by revision, so a
+second run reads nothing from the network and writes the same bytes.
+"""
+
+from __future__ import annotations
+
+import importlib
+import pkgutil
+import sys
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+_SCRIPTS_DIR = str(Path(__file__).resolve().parents[2])
+if _SCRIPTS_DIR not in sys.path:
+    sys.path.insert(0, _SCRIPTS_DIR)
+
+import upstream  # noqa: E402
+from safeparse import yaml_load  # noqa: E402
+
+CACHE_DIR = upstream.CACHE_DIR
+
+# Keys a setup file carries, in the order they are written.
+SETUP_KEYS = (
+    "setup", "name", "platform", "sources", "roms_root", "case_sensitive",
+    "case_ref", "skip", "hidden", "media", "notes", "folders",
+)
+ROW_KEYS = (
+    "folder", "name", "aliases", "extensions", "core", "emulator", "source_ref",
+    "board_defaults", "arcade", "note",
+)
+
+
+class SourceError(RuntimeError):
+    """A pinned file is absent, or its content no longer has the shape read."""
+
+
+@dataclass
+class Fetcher:
+    """Reads pinned files of one setup's declared sources."""
+
+    sources: dict[str, dict]
+    cache_dir: str = CACHE_DIR
+    offline: bool = False
+
+    def _repo(self, key: str) -> tuple[upstream.Repo, str]:
+        if key not in self.sources:
+            raise SourceError(f"source {key!r} is not declared")
+        spec = self.sources[key]
+        repo = upstream.parse_repo(spec["repo"])
+        if repo is None:
+            raise SourceError(f"source {key!r}: unsupported forge {spec['repo']}")
+        return repo, spec["commit"]
+
+    def text(self, key: str, path: str) -> str:
+        """One file at the pinned commit, or SourceError when absent."""
+        repo, sha = self._repo(key)
+        lines = upstream.fetch_file(repo, sha, path, self.cache_dir, self.offline)
+        if lines is None:
+            raise SourceError(f"{repo.slug}@{sha[:8]}:{path} does not exist")
+        return "\n".join(lines) + "\n"
+
+    def optional_text(self, key: str, path: str) -> str | None:
+        repo, sha = self._repo(key)
+        lines = upstream.fetch_file(repo, sha, path, self.cache_dir, self.offline)
+        return None if lines is None else "\n".join(lines) + "\n"
+
+    def tree(self, key: str, prefix: str = "") -> list[str]:
+        """Every blob path under a prefix at the pinned commit."""
+        repo, sha = self._repo(key)
+        if repo.family == "gitlab" and prefix.strip("/"):
+            return self._gitlab_subtree(repo, sha, prefix)
+        paths, truncated = upstream.list_tree(repo, sha, self.cache_dir, self.offline)
+        if truncated:
+            raise SourceError(f"{repo.slug}@{sha[:8]}: the forge truncated the tree")
+        return sorted(p for p in paths if p.startswith(prefix))
+
+    def entries(self, key: str, directory: str) -> list[str]:
+        """Paths of the files and folders directly under one directory."""
+        repo, sha = self._repo(key)
+        if repo.family != "gitlab":
+            prefix = directory.strip("/") + "/"
+            return sorted({prefix + p[len(prefix):].split("/", 1)[0] for p in self.tree(key, prefix)})
+        return self._gitlab_listing(repo, sha, directory, recursive=False, blobs=False)
+
+    def _gitlab_subtree(self, repo: upstream.Repo, sha: str, prefix: str) -> list[str]:
+        """GitLab pages its tree 100 entries at a time and Recalbox's whole
+        tree outruns the page cap: list only the directory asked for."""
+        paths = self._gitlab_listing(repo, sha, prefix, recursive=True, blobs=True)
+        return sorted(p for p in paths if p.startswith(prefix))
+
+    def _gitlab_listing(
+        self, repo: upstream.Repo, sha: str, directory: str, *, recursive: bool, blobs: bool
+    ) -> list[str]:
+        directory = directory.strip("/")
+        paths: list[str] = []
+        for page in range(1, upstream.MAX_TREE_PAGES + 1):
+            url = (
+                f"{repo.api_base}/projects/{upstream._project(repo)}/repository/tree"
+                f"?ref={sha}&path={directory}&recursive={'true' if recursive else 'false'}"
+                f"&per_page={upstream.TREE_PAGE_SIZE}&page={page}"
+            )
+            payload = upstream._api(url, self.cache_dir, self.offline)
+            if not isinstance(payload, list):
+                raise SourceError(f"{repo.slug}@{sha[:8]}: {directory} could not be listed")
+            paths.extend(
+                entry["path"] for entry in payload
+                if isinstance(entry, dict) and entry.get("path")
+                and (entry.get("type") == "blob" or not blobs)
+            )
+            if len(payload) < upstream.TREE_PAGE_SIZE:
+                return sorted(paths)
+        raise SourceError(f"{repo.slug}@{sha[:8]}: {directory} outruns the page cap")
+
+
+Parser = Callable[[dict, Fetcher], list[dict]]
+FORMATS: dict[str, Parser] = {}
+
+
+def register(name: str) -> Callable[[Parser], Parser]:
+    def wrap(func: Parser) -> Parser:
+        if name in FORMATS:
+            raise ValueError(f"format {name!r} registered twice")
+        FORMATS[name] = func
+        return func
+
+    return wrap
+
+
+def load_formats() -> dict[str, Parser]:
+    """Import every parser module of this package once."""
+    if not FORMATS:
+        for info in pkgutil.iter_modules([str(Path(__file__).parent)]):
+            if not info.name.startswith("_"):
+                importlib.import_module(f"{__name__}.{info.name}")
+    return FORMATS
+
+
+def load_sources(directory: Path) -> dict[str, dict]:
+    """Each setup's declared sources and facts, keyed by its file name."""
+    out: dict[str, dict] = {}
+    for path in sorted(Path(directory).glob("*.yml")):
+        with path.open(encoding="utf-8") as handle:
+            out[path.stem] = yaml_load(handle) or {}
+    return out
+
+
+def _clean_row(row: dict) -> dict:
+    unknown = set(row) - set(ROW_KEYS)
+    if unknown:
+        raise ValueError(f"folder {row.get('folder')!r}: unknown keys {sorted(unknown)}")
+    out = {key: row[key] for key in ROW_KEYS if row.get(key) not in (None, [], {}, "")}
+    if out.get("name") == out["folder"]:
+        del out["name"]
+    if "extensions" in out:
+        out["extensions"] = sorted({e.lower() for e in out["extensions"]})
+    if "aliases" in out:
+        out["aliases"] = sorted(set(out["aliases"]) - {out["folder"]})
+        if not out["aliases"]:
+            del out["aliases"]
+    return out
+
+
+def scrape_setup(setup: str, spec: dict, fetcher: Fetcher) -> dict:
+    """The setup document: declared facts, then the parsed folders."""
+    formats = load_formats()
+    fmt = spec.get("format")
+    rows: list[dict] = []
+    if fmt:
+        if fmt not in formats:
+            raise SourceError(f"{setup}: no parser for format {fmt!r}")
+        rows = formats[fmt](spec, fetcher)
+        if not rows and not spec.get("allow_empty"):
+            raise SourceError(f"{setup}: format {fmt} read no folder")
+    folders: dict[str, dict] = {}
+    for row in rows:
+        clean = _clean_row(row)
+        key = clean["folder"]
+        if key in folders:
+            raise SourceError(f"{setup}: folder {key!r} read twice")
+        folders[key] = clean
+    doc = {"setup": setup, **{k: spec[k] for k in SETUP_KEYS[1:-1] if k in spec}}
+    doc["folders"] = [folders[k] for k in sorted(folders, key=lambda f: (f.lower(), f))]
+    return doc

scripts/scraper/romfolders/__main__.py

@@ -0,0 +1,75 @@
+"""Write platforms/romfolders/<setup>.yml from each setup's pinned source.
+
+    python -m scripts.scraper.romfolders --all
+    python -m scripts.scraper.romfolders --setup batocera --dry-run
+    python -m scripts.scraper.romfolders --all --offline   # cache only
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+import yaml
+
+from . import Fetcher, SourceError, load_sources, scrape_setup
+from upstream import UpstreamError  # noqa: E402  (the package puts scripts/ on sys.path)
+from artifacts import write_text_atomic  # noqa: E402
+from ..base_scraper import _PlatformDumper
+
+DEFAULT_DIR = Path("platforms") / "romfolders"
+
+
+def dump_setup(doc: dict) -> str:
+    return yaml.dump(
+        doc, Dumper=_PlatformDumper, sort_keys=False, allow_unicode=True, width=100
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.splitlines()[0])
+    which = parser.add_mutually_exclusive_group(required=True)
+    which.add_argument("--setup", action="append", help="setup id (repeatable)")
+    which.add_argument("--all", action="store_true", help="every setup of the sources file")
+    which.add_argument("--list", action="store_true", help="list the setups and their format")
+    parser.add_argument("--sources", type=Path, default=DEFAULT_DIR / "_sources",
+                        help="the setups, their repositories and pinned revisions")
+    parser.add_argument("--output-dir", type=Path, default=DEFAULT_DIR,
+                        help=f"where <setup>.yml is written (default {DEFAULT_DIR})")
+    parser.add_argument("--offline", action="store_true", help="read the revision cache only")
+    parser.add_argument("--dry-run", action="store_true", help="print counts, write nothing")
+    args = parser.parse_args()
+
+    sources = load_sources(args.sources)
+    if args.list:
+        for setup, spec in sources.items():
+            print(f"{setup:16} {spec.get('format') or '-':14} {spec.get('name', '')}")
+        return 0
+    wanted = list(sources) if args.all else args.setup
+    unknown = [s for s in wanted if s not in sources]
+    if unknown:
+        print(f"Error: unknown setup(s): {', '.join(unknown)}", file=sys.stderr)
+        return 2
+    failed = 0
+    for setup in wanted:
+        spec = sources[setup]
+        fetcher = Fetcher(spec.get("sources", {}), offline=args.offline)
+        try:
+            doc = scrape_setup(setup, spec, fetcher)
+        except (SourceError, UpstreamError, ValueError, KeyError, OSError) as exc:
+            print(f"FAIL {setup}: {exc}", file=sys.stderr)
+            failed += 1
+            continue
+        print(f"{setup}: {len(doc['folders'])} folders")
+        if not args.dry_run:
+            args.output_dir.mkdir(parents=True, exist_ok=True)
+            target = args.output_dir / f"{setup}.yml"
+            text = dump_setup(doc)
+            if not target.exists() or target.read_text(encoding="utf-8") != text:
+                write_text_atomic(str(target), text)
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/scraper/romfolders/card.py

@@ -0,0 +1,628 @@
+"""Rom folder parsers for handheld card OSes and the frontends without a menu.
+
+Each parser reads the system list the OS ships at the pinned commit and
+returns one row per folder, in the OS's own spelling:
+
+- `onion`: Onion's optional packages, `Emu/<SYS>/config.json` and the
+  `launch.sh` beside it. RetroArch-section packages (`RApp/`) add cores on the
+  same folder, the stock menu starts the `Emu/` one.
+- `spruce`: spruceOS `Emu/<SYS>/config.json`, `default_emulator`.
+- `crossmix`: CrossMix-OS `Emus/<SYS>/config.json`, first `launchlist` entry
+  with a script, as `load_launcher.sh` picks it.
+- `koriki`: SimpleMenu section groups, first launcher of `execs`.
+- `allium`: Allium `consoles.toml`, first entry of `cores`.
+- `minui`: MinUI, MyMinUI and NextUI paks. A Roms folder starts the pak named
+  by the text in its last parentheses (`getEmuName`), so the folder of a row
+  is the one the skeleton ships, else the tag alone.
+- `plumos`: plumOS `systems.json`, `directory_aliases` by priority.
+- `muos`: muOS `core.json`, every `friendly` name of a system.
+- `gmenu2x`: MiyooCFW, the folders its image ships and the gmenu2x links
+  whose text a repository holds.
+- `mister`: MiSTer, one `games/<core>/` folder per core the distribution ships.
+- `romm`: RomM, one library folder per platform slug it plays or keeps BIOS for.
+- `bizhawk`: BizHawk's per-system base folders.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Iterable
+from configparser import ConfigParser
+from pathlib import PurePosixPath
+
+from . import Fetcher, SourceError, register
+
+_LIBRETRO = re.compile(r"([A-Za-z0-9_+.\-${}]+?)_libretro\.so")
+_EMU_EXE = re.compile(r"^\s*EMU_EXE=[\"']?([A-Za-z0-9_+.\-]+)", re.M)
+
+
+def line_of(text: str, needle: str, start: int = 0) -> int:
+    """1-based line of the first needle at or after start, 0 when absent."""
+    idx = text.find(needle, start)
+    return text.count("\n", 0, idx) + 1 if idx >= 0 else 0
+
+
+def ref(key: str, path: str, line: int | str | None = None) -> str:
+    """A source_ref part. A space in a path is written %20, as in a URL:
+    the ref grammar ends a path at the first space."""
+    path = path.replace("%", "%25").replace(" ", "%20")
+    return f"{key} {path}:{line}" if line else f"{key} {path}"
+
+
+def extensions(raw: str | Iterable[str] | None) -> list[str]:
+    """Lowercased extensions with the dot, from a list or a `|`/`,` string."""
+    items = re.split(r"[|,]", raw) if isinstance(raw, str) else list(raw or [])
+    out: list[str] = []
+    for item in items:
+        ext = str(item).strip().lower().lstrip(".")
+        if ext and f".{ext}" not in out:
+            out.append(f".{ext}")
+    return out
+
+
+def _shell_value(script: str, name: str) -> str | None:
+    match = re.search(rf"^\s*{re.escape(name)}=[\"']?([A-Za-z0-9_+.\-]+)", script, re.M)
+    return match.group(1) if match else None
+
+
+def libretro_core(script: str) -> str | None:
+    """The core a launch script names: `<core>_libretro.so`, a shell variable
+    assigned in the same script, or `EMU_EXE=`."""
+    for match in _LIBRETRO.finditer(script):
+        name = match.group(1).split("/")[-1]
+        if "$" not in name:
+            return name
+        var = re.fullmatch(r"\$\{?(\w+)\}?", name)
+        if var:
+            value = _shell_value(script, var.group(1))
+            if value:
+                return value
+    exe = _EMU_EXE.search(script)
+    return exe.group(1) if exe else None
+
+
+def _json(fetcher: Fetcher, key: str, path: str) -> tuple[dict, str]:
+    text = fetcher.text(key, path)
+    try:
+        return json.loads(text), text
+    except ValueError as exc:
+        raise SourceError(f"{path}: {exc}") from exc
+
+
+def _rompath_folder(rompath: str) -> str:
+    """The folder under Roms/ a stock-menu config.json points at."""
+    path = rompath.strip().replace("\\", "/")
+    for prefix in ("../../Roms/", "/mnt/SDCARD/Roms/"):
+        if path.startswith(prefix):
+            return path[len(prefix):].strip("/")
+    raise SourceError(f"rompath {rompath!r} is not under Roms/")
+
+
+# Onion
+
+@register("onion")
+def parse_onion(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    pattern = re.compile(r"^static/packages/(Emu|RApp)/([^/]+)/\1/([^/]+)/config\.json$")
+    packages: dict[str, list[dict]] = {}
+    for path in fetcher.tree(key, "static/packages/"):
+        match = pattern.match(path)
+        if not match:
+            continue
+        layer, package = match.group(1), match.group(2)
+        config, text = _json(fetcher, key, path)
+        script_path = f"{path.rsplit('/', 1)[0]}/{config.get('launch', 'launch.sh')}"
+        script = fetcher.optional_text(key, script_path) or ""
+        packages.setdefault(_rompath_folder(config.get("rompath", "")), []).append({
+            "layer": layer, "package": package, "config": config,
+            "core": libretro_core(script), "ref": ref(key, path, line_of(text, '"rompath"')),
+            "script": script_path,
+        })
+    rows = []
+    for folder, found in packages.items():
+        found.sort(key=lambda p: (p["layer"] != "Emu", p["package"]))
+        emu = [p for p in found if p["layer"] == "Emu"]
+        first = emu[0] if emu else found[0]
+        others = [p["package"] for p in found if p is not first]
+        row = {
+            "folder": folder,
+            "extensions": extensions(first["config"].get("extlist")),
+            "source_ref": first["ref"],
+        }
+        if emu:
+            row["core"] = first["core"]
+            row["emulator"] = "retroarch" if first["core"] else "standalone"
+            note = f"package '{first['package']}' (Emu/, the stock menu entry)"
+            if not first["core"]:
+                note += f", launches {first['script']}"
+        else:
+            note = "only RetroArch-section packages (RApp/), the user picks the core"
+        if others:
+            note += f". Other packages on this folder: {', '.join(others)}"
+        row["note"] = note
+        rows.append(row)
+    return rows
+
+
+# spruceOS
+
+@register("spruce")
+def parse_spruce(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    skip = set(params.get("skip", []))
+    rows = []
+    for path in fetcher.tree(key, "Emu/"):
+        parts = path.split("/")
+        if len(parts) != 3 or parts[2] != "config.json" or parts[1] in skip:
+            continue
+        config, text = _json(fetcher, key, path)
+        notes = []
+        if config.get("devices"):
+            notes.append(f"only on devices {', '.join(map(str, config['devices']))}")
+        if config.get("ignoreList"):
+            notes.append(f"ignoreList {', '.join(map(str, config['ignoreList']))}")
+        rows.append({
+            "folder": parts[1],
+            "aliases": [str(a) for a in config.get("alternativeFolderNames") or []],
+            "extensions": extensions(config.get("extlist")),
+            "core": config.get("default_emulator"),
+            "source_ref": ref(key, path, line_of(text, '"default_emulator"') or 1),
+            "note": ". ".join(notes),
+        })
+    return rows
+
+
+# CrossMix-OS
+
+@register("crossmix")
+def parse_crossmix(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    rows = []
+    for path in fetcher.tree(key, "Emus/"):
+        parts = path.split("/")
+        if len(parts) != 3 or parts[2] != "config.json" or parts[1].startswith("_"):
+            continue
+        config, text = _json(fetcher, key, path)
+        launchers = [e for e in config.get("launchlist") or [] if e.get("launch")]
+        script_name = launchers[0]["launch"] if launchers else config.get("launch", "launch.sh")
+        script_path = f"Emus/{parts[1]}/{script_name}"
+        script = fetcher.optional_text(key, script_path) or ""
+        core = libretro_core(script)
+        note = f"default launcher {launchers[0].get('name') if launchers else script_name} ({script_path})"
+        folder = _rompath_folder(config.get("rompath", ""))
+        rows.append({
+            "folder": folder,
+            "name": parts[1] if parts[1] != folder else None,
+            "extensions": extensions(config.get("extlist")),
+            "core": core,
+            "emulator": "retroarch" if core else "standalone",
+            "source_ref": ref(key, path, line_of(text, '"rompath"') or 1),
+            "note": note if core else f"{note}, names no libretro core",
+        })
+    return rows
+
+
+# Koriki
+
+@register("koriki")
+def parse_koriki(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    path = params["sections"]
+    text = fetcher.text(key, path)
+    parser = ConfigParser(interpolation=None, strict=False)
+    parser.optionxform = str  # type: ignore[assignment]
+    parser.read_string(text)
+    skip = set(params.get("skip", []))
+    rows = []
+    for section in parser.sections():
+        if section in skip:
+            continue
+        body = parser[section]
+        dirs = [d.strip() for d in body.get("romDirs", "").split(",") if d.strip()]
+        if not dirs:
+            continue
+        rel = [d.replace(params["roms_prefix"], "").strip("/") for d in dirs]
+        execs = [PurePosixPath(e.strip()).name for e in body.get("execs", "").split(",") if e.strip()]
+        core = None
+        if execs:
+            launcher = fetcher.optional_text(key, f"{params['launchers']}/{execs[0]}")
+            core = libretro_core(launcher or "")
+        note = f"launchers {', '.join(execs)}"
+        if len(rel) > 1 or "/" in rel[0]:
+            note += f". romDirs {', '.join(rel)}"
+        rows.append({
+            "folder": rel[0].split("/")[0],
+            "name": section,
+            "extensions": extensions(body.get("romExts")),
+            "core": core or (execs[0] if execs else None),
+            "emulator": "retroarch" if core else "standalone",
+            "source_ref": ref(key, path, line_of(text, f"[{section}]")),
+            "note": note,
+        })
+    return rows
+
+
+# Allium
+
+_TOML_KEY = r'(?:"[^"]*"|[A-Za-z0-9_\-]+)'
+
+
+def _toml_value(raw: str) -> object:
+    raw = raw.strip()
+    if raw.startswith('"') and raw.endswith('"'):
+        return raw[1:-1]
+    if raw.startswith("["):
+        return re.findall(r'"([^"]*)"', raw)
+    if raw in ("true", "false"):
+        return raw == "true"
+    if re.fullmatch(r"-?\d+", raw):
+        return int(raw)
+    raise SourceError(f"TOML value {raw!r} is outside the subset Allium's files use")
+
+
+def parse_toml_subset(text: str) -> dict:
+    """The TOML Allium writes: `[[array]]`, `[a.b]` tables, string, integer,
+    boolean and string-array values, arrays that may span lines. Anything
+    else is refused rather than guessed. Each table records the line it
+    starts on under `__line__`."""
+    root: dict = {}
+    current: dict = root
+    lines = text.splitlines()
+    i = 0
+    while i < len(lines):
+        line = lines[i].split("#", 1)[0].strip() if '"' not in lines[i] else lines[i].strip()
+        i += 1
+        if not line or line.startswith("#"):
+            continue
+        if line.startswith("[["):
+            name = line.strip("[]").strip()
+            current = {"__line__": i}
+            root.setdefault(name, []).append(current)
+            continue
+        if line.startswith("["):
+            current = root
+            for part in re.findall(_TOML_KEY, line.strip("[]")):
+                current = current.setdefault(part.strip('"'), {})
+            current["__line__"] = i
+            continue
+        match = re.match(rf"({_TOML_KEY})\s*=\s*(.*)$", line)
+        if not match:
+            raise SourceError(f"TOML line {i}: {line!r} is outside the subset Allium's files use")
+        value = match.group(2)
+        while value.count("[") > value.count("]") and i < len(lines):
+            value += lines[i].strip()
+            i += 1
+        current[match.group(1).strip('"')] = _toml_value(value)
+    return root
+
+
+@register("allium")
+def parse_allium(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    consoles = parse_toml_subset(fetcher.text(key, params["consoles"]))["consoles"]
+    cores = parse_toml_subset(fetcher.text(key, params["cores"])).get("cores", {})
+    rows = []
+    for console in consoles:
+        patterns = [str(p) for p in console.get("patterns") or []]
+        if not patterns:
+            continue
+        first = (console.get("cores") or [None])[0]
+        core_def = cores.get(first, {}) if first else {}
+        core = core_def.get("retroarch") if isinstance(core_def, dict) else None
+        rows.append({
+            "folder": patterns[0],
+            "name": console["name"],
+            "aliases": patterns[1:],
+            "extensions": extensions(console.get("extensions")),
+            "core": core or first,
+            "emulator": "retroarch" if core else "standalone",
+            "source_ref": ref(key, params["consoles"], console["__line__"]),
+            "note": "a folder named any pattern, or ending in '(<pattern>)', is this console. A file's own extension wins over its folder",
+        })
+    return rows
+
+
+# MinUI, MyMinUI, NextUI
+
+_PAK = re.compile(r"^skeleton/(?:[^/]+/)+(?:Emus|paks/Emus)/(?:[^/]+/)?([^/]+)\.pak/launch\.sh$")
+
+
+def _pak_device(path: str) -> str:
+    """The device directory a pak sits under (`Emus/<device>/X.pak`,
+    `SYSTEM/<device>/paks/Emus/X.pak`)."""
+    parts = path.split("/")
+    if "paks" in parts:
+        return parts[parts.index("paks") - 1]
+    idx = parts.index("Emus")
+    return parts[idx + 1] if not parts[idx + 1].endswith(".pak") else parts[1]
+
+
+@register("minui")
+def parse_minui(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    tree = fetcher.tree(key, "skeleton/")
+    shipped: dict[str, str] = {}
+    for path in tree:
+        match = re.match(r"^skeleton/[^/]+/Roms/([^/]+)/", path)
+        if match:
+            name = match.group(1)
+            tag = re.search(r"\(([^()]*)\)\s*$", name)
+            shipped.setdefault(tag.group(1) if tag else name, name)
+    paks: dict[str, dict[str, tuple[str, str]]] = {}
+    for path in tree:
+        match = _PAK.match(path)
+        if not match:
+            continue
+        script = fetcher.text(key, path)
+        core = libretro_core(script)
+        line = line_of(script, "EMU_EXE=") or line_of(script, "_libretro.so") or 1
+        paks.setdefault(match.group(1), {})[_pak_device(path)] = (core or "", ref(key, path, line))
+    rows = []
+    for tag, devices in sorted(paks.items()):
+        counts: dict[str, int] = {}
+        for core, _ in devices.values():
+            counts[core] = counts.get(core, 0) + 1
+        core = min(counts, key=lambda c: (-counts[c], c == "", c))
+        default_ref = next(r for c, r in sorted(devices.values()) if c == core)
+        boards = {
+            device: {"core": c, "source_ref": r}
+            for device, (c, r) in sorted(devices.items()) if c != core and c
+        }
+        folder = shipped.get(tag, tag)
+        rows.append({
+            "folder": folder,
+            "name": tag,
+            "aliases": [tag] if folder != tag else [],
+            "core": core or None,
+            "emulator": "minarch" if core else "standalone",
+            "source_ref": default_ref,
+            "board_defaults": boards,
+            "note": (f"any folder named {tag} or ending in '({tag})' starts {tag}.pak. "
+                     f"Devices: {', '.join(sorted(devices))}"),
+        })
+    return rows
+
+
+# plumOS
+
+@register("plumos")
+def parse_plumos(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    text = fetcher.text(key, params["path"])
+    systems = json.loads(text)["systems"]
+    taken: dict[str, str] = {}
+    rows = []
+    pos = 0
+    for system in systems:
+        needle = f'"id": "{system["id"]}"'
+        line = line_of(text, needle, pos)
+        pos = max(pos, text.find(needle, pos) + 1)
+        aliases = [a["name"] for a in sorted(system.get("directory_aliases") or [],
+                                             key=lambda a: a.get("priority", 0))]
+        free = [a for a in aliases if a not in taken]
+        if not free:
+            continue
+        folder = free[0]
+        taken[folder] = system["id"]
+        kind, _, core = (system.get("default_launch_profile") or "").partition(":")
+        notes = [] if system.get("enabled") else ["disabled by default"]
+        shared = [f"{a} (also {taken[a]})" for a in aliases if a in taken and taken[a] != system["id"]]
+        if shared:
+            notes.append(f"shares {', '.join(shared)}, the extension decides")
+        rows.append({
+            "folder": folder,
+            "name": system["id"],
+            "aliases": [a for a in aliases if a != folder],
+            "extensions": extensions(system.get("extensions")),
+            "core": core or None,
+            "emulator": kind or None,
+            "source_ref": ref(key, params["path"], line),
+            "note": ". ".join(notes),
+        })
+    return rows
+
+
+# muOS
+
+def _muos_norm(name: str) -> str:
+    """The key muOS looks a folder up by (common.c: lowercase, drop ' -_+')."""
+    return re.sub(r"[ \-_+]", "", name.lower())
+
+
+@register("muos")
+def parse_muos(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    text = fetcher.text(key, params["path"])
+    data = json.loads(text)
+    rows = []
+    for system, entry in data.items():
+        friendly = [str(f) for f in entry.get("friendly") or []]
+        if not friendly:
+            continue
+        want = _muos_norm(entry.get("catalogue") or system)
+        folder = next((f for f in friendly if f == want), friendly[0])
+        default = entry.get("default")
+        libretro = entry.get("libretro") or {}
+        so = (libretro.get(default) or {}).get("core") if default in libretro else None
+        core = so[: -len("_libretro.so")] if so and so.endswith("_libretro.so") else default
+        rows.append({
+            "folder": folder,
+            "name": system,
+            "aliases": friendly,
+            "core": core,
+            "emulator": "retroarch" if so else "external",
+            "source_ref": ref(key, params["path"], line_of(text, json.dumps(system) + ":")),
+            "note": "a folder matches when its name, lowercased without ' -_+', is one of the friendly names",
+        })
+    return rows
+
+
+# MiyooCFW
+
+@register("gmenu2x")
+def parse_gmenu2x(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    rows: dict[str, dict] = {}
+    for link in params.get("links", []):
+        text = fetcher.text(link["source"], link["path"])
+        fields = dict(
+            line.split("=", 1) for line in text.splitlines() if "=" in line
+        )
+        selector = fields.get("selectordir", "").rstrip("/")
+        prefix = params["link_roms_prefix"]
+        if not selector.startswith(prefix):
+            raise SourceError(f"{link['path']}: selectordir {selector!r} is not under {prefix}")
+        folder = selector[len(prefix):].strip("/")
+        rows[folder] = {
+            "folder": folder,
+            "extensions": extensions(fields.get("selectorfilter")),
+            "core": link["core"],
+            "emulator": "standalone",
+            "source_ref": ref(link["source"], link["path"], line_of(text, "selectordir=")),
+            "note": f"gmenu2x link of the {link['core']} package",
+        }
+    key = params["source"]
+    for path in fetcher.tree(key, params["shipped_roms"] + "/"):
+        folder = path[len(params["shipped_roms"]) + 1:].split("/")
+        if len(folder) < 2 or folder[0] in rows:
+            continue
+        rows[folder[0]] = {
+            "folder": folder[0],
+            "source_ref": ref(key, f"{params['shipped_roms']}/{folder[0]}"),
+            "note": "folder the image ships. Its emulator link lives in a release package no repository holds",
+        }
+    return list(rows.values())
+
+
+# MiSTer
+
+@register("mister")
+def parse_mister(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    arcade = set(params.get("arcade", []))
+    folders = sorted({p.split("/")[1] for p in fetcher.tree(key, "games/") if p.count("/") >= 2})
+    return [
+        {
+            "folder": folder,
+            "emulator": "mister-core",
+            "source_ref": ref(key, f"games/{folder}/"),
+            "arcade": folder in arcade or None,
+            "note": params["arcade_note"] if folder in arcade else None,
+        }
+        for folder in folders
+        if folder not in params.get("skip", [])
+    ]
+
+
+# RomM
+
+@register("romm")
+def parse_romm(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    text = fetcher.text(key, params["cores_map"])
+    lines = text.splitlines()
+    maps: dict[str, dict[str, tuple[list[str], int]]] = {}
+    block = None
+    entry = None
+    for number, line in enumerate(lines, 1):
+        start = re.match(r"const (_EJS_(?:NIGHTLY_)?CORES_MAP)\b", line)
+        if start:
+            block = start.group(1)
+            continue
+        if block and line.startswith("}"):
+            block = None
+            continue
+        if not block:
+            continue
+        head = re.match(r'\s+"?([\w-]+)"?:\s*\[(.*)$', line)
+        if head:
+            entry = (head.group(1), number)
+            maps.setdefault(block, {})[entry[0]] = (re.findall(r'"([\w-]+)"', head.group(2)), number)
+            if "]" in head.group(2):
+                entry = None
+        elif entry:
+            maps[block][entry[0]][0].extend(re.findall(r'"([\w-]+)"', line))
+            if "]" in line:
+                entry = None
+    stable = maps.get("_EJS_CORES_MAP", {})
+    nightly = maps.get("_EJS_NIGHTLY_CORES_MAP", {})
+    if not stable:
+        raise SourceError(f"{params['cores_map']}: _EJS_CORES_MAP not found")
+    bios = json.loads(fetcher.text(key, params["bios_fixture"]))
+    bios_slugs = {name.split(":", 1)[0] for name in bios}
+    rows = []
+    for slug in sorted(set(stable) | set(nightly) | bios_slugs):
+        found = stable.get(slug) or nightly.get(slug)
+        if found:
+            note = None if slug in stable else "core only with EmulatorJS nightly cores"
+            rows.append({"folder": slug, "core": found[0][0], "emulator": "emulatorjs",
+                         "source_ref": ref(key, params["cores_map"], found[1]), "note": note})
+        else:
+            rows.append({"folder": slug, "source_ref": ref(key, params["bios_fixture"]),
+                         "note": "no EmulatorJS core, RomM keeps BIOS for it"})
+    return rows
+
+
+# BizHawk
+
+@register("bizhawk")
+def parse_bizhawk(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params["source"]
+    entries = fetcher.text(key, params["paths"])
+    loader = fetcher.text(key, params["rom_loader"])
+    config = fetcher.text(key, params["config"])
+    combined = {
+        m.group(1): re.findall(r"VSystemID\.Raw\.(\w+)", m.group(2))
+        for m in re.finditer(r"string (COMBINED_SYSIDS_\w+) = string\.Join\(\"_\", ([^)]*)\)", entries)
+    }
+    ext_fields: dict[str, tuple[list[str], int]] = {}
+    for number, line in enumerate(loader.splitlines(), 1):
+        clean = re.sub(r"/\*.*?\*/", "", line)
+        match = re.search(r"IReadOnlyCollection<string> (\w+) = new\[\] \{ (.*?) \}", clean)
+        if match:
+            ext_fields[match.group(1)] = (re.findall(r'"([^"]+)"', match.group(2)), number)
+    preferred: dict[str, str] = {}
+    picker = config[config.find("CorePickerUIData"):]
+    for match in re.finditer(r"\(\[([^\]]+)\],\s*\[\s*CoreNames\.(\w+)", picker):
+        for sid in re.findall(r"VSystemID\.Raw\.(\w+)", match.group(1)):
+            preferred.setdefault(sid, match.group(2))
+    rows = []
+    for number, line in enumerate(entries.splitlines(), 1):
+        match = re.search(
+            r"CommonEntriesFor\((?:VSystemID\.Raw\.(\w+)|(COMBINED_SYSIDS_\w+)), "
+            r"basePath: Path\.Combine\(\"\.\", \"([^\"]+)\"\)", line)
+        if not match:
+            continue
+        sids = [match.group(1)] if match.group(1) else combined.get(match.group(2), [])
+        exts: list[str] = []
+        refs = [ref(key, params["paths"], number)]
+        for sid in sids:
+            if sid in ext_fields:
+                exts.extend(ext_fields[sid][0])
+                refs.append(ref(key, params["rom_loader"], ext_fields[sid][1]))
+        cores = [preferred[s] for s in sids if s in preferred]
+        note = "the open dialog starts here, nothing is scanned"
+        if cores:
+            note += f". Default core {cores[0]} (CorePickerUIData, CoreNames[0])"
+        rows.append({
+            "folder": match.group(3),
+            "name": sids[0] if sids else None,
+            "aliases": sids[1:],
+            "extensions": extensions(exts),
+            "emulator": "EmuHawk",
+            "source_ref": "; ".join(refs),
+            "note": note,
+        })
+    return rows

scripts/scraper/romfolders/es.py

@@ -0,0 +1,781 @@
+"""EmulationStation family: the system lists each OS builds its menu from.
+
+Every parser returns one row per folder under the roms root, with the
+extensions the list accepts, the core the OS starts by default and the
+line of the list that says so. Systems the list keeps outside the roms root
+(an absolute path elsewhere) or marks as no game system are left out: the
+installer never scans them.
+
+Formats, by the shape of the source:
+
+- `batocera-es`: es_systems.yml plus configgen-defaults.yml and its per-board
+  files (Batocera, KNULLI).
+- `es-xml`: a static es_systems.cfg (AmberELEC, CHAos, PAN4ELEC, ArkOS,
+  dArkOS, RetroBat).
+- `es-json`: EmuELEC's es_systems.json, turned into the XML at build time.
+- `elec-conf`: config/emulators/<system>.conf plus the add_emu_core and
+  add_es_system calls of the emulators package (ROCKNIX, JELOS, KOLOS,
+  UnofficialOS, ArchR).
+- `esde`: ES-DE's es_systems.xml, with the alternative emulators and custom
+  systems an installer writes over it (EmuDeck, RetroDECK).
+- `recalbox`: one system.ini per system (Recalbox).
+- `retropie`: platforms.cfg and the addEmulator calls of the scriptmodules.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shlex
+from collections.abc import Iterator
+
+from safeparse import parse_untrusted_xml, yaml_load_upstream
+
+from . import Fetcher, SourceError, register
+
+
+def _ref(key: str, path: str, *lines: int | str | None) -> str:
+    kept = [str(n) for n in lines if n not in (None, "")]
+    return f"{key} {path}:{','.join(kept)}" if kept else f"{key} {path}"
+
+
+def _board(core: str | None, emulator: str | None, source_ref: str) -> dict:
+    out = {"core": core or emulator, "emulator": emulator, "source_ref": source_ref}
+    return {k: str(v) for k, v in out.items() if v}
+
+
+def _top_key_lines(text: str) -> dict[str, int]:
+    """First line of each top-level YAML key."""
+    out: dict[str, int] = {}
+    for number, line in enumerate(text.splitlines(), 1):
+        match = re.match(r"^([A-Za-z0-9_+.\-]+):", line)
+        if match:
+            out.setdefault(match.group(1), number)
+    return out
+
+
+def _strip_prefix(path: str, prefixes: list[str]) -> str | None:
+    """The folder under the roms root, or None for a path outside it."""
+    for prefix in prefixes:
+        if path.startswith(prefix):
+            rest = path[len(prefix):].strip("/\\")
+            return rest.replace("\\", "/") or None
+    return None
+
+
+def _extensions(raw: str | list | None) -> list[str]:
+    words = raw.split() if isinstance(raw, str) else [str(w) for w in raw or []]
+    return [w if w.startswith(".") else f".{w}" for w in words if w]
+
+
+# ---------------------------------------------------------------------------
+# Batocera and KNULLI
+
+
+def _yaml_or_link(fetcher: Fetcher, key: str, directory: str, name: str) -> tuple[dict, str, str]:
+    """A configgen file, following a git symlink the raw route serves as text."""
+    path = f"{directory}/{name}"
+    text = fetcher.text(key, path)
+    data = yaml_load_upstream(text)
+    if isinstance(data, str) and data.strip().endswith(".yml"):
+        path = f"{directory}/{data.strip()}"
+        text = fetcher.text(key, path)
+        data = yaml_load_upstream(text)
+    if not isinstance(data, dict):
+        raise SourceError(f"{path} is not a mapping")
+    return data, text, path
+
+
+@register("batocera-es")
+def batocera_es(spec: dict, fetcher: Fetcher) -> list[dict]:
+    """es_systems.yml read the way batocera-es-system writes the XML.
+
+    A system with no emulator and no `force` is never written
+    (es_systems.py `_system_dict_to_xml`), `path` defaults to the key and an
+    absolute path leaves the roms root. The default core is configgen's:
+    the board file first, then configgen-defaults.yml (shared.py
+    ConfiggenDefaults.get).
+    """
+    params = spec["params"]
+    key = params.get("source", "os")
+    es_path = params["es_systems"]
+    es_text = fetcher.text(key, es_path)
+    systems = yaml_load_upstream(es_text)
+    if not isinstance(systems, dict):
+        raise SourceError(f"{es_path} is not a mapping")
+    es_lines = _top_key_lines(es_text)
+    directory = params["configgen_dir"]
+    defaults, def_text, def_path = _yaml_or_link(fetcher, key, directory, "configgen-defaults.yml")
+    def_lines = _top_key_lines(def_text)
+    boards: dict[str, tuple[dict, dict[str, int], str]] = {}
+    for path in fetcher.tree(key, f"{directory}/configgen-defaults-"):
+        name = path.rsplit("/", 1)[-1]
+        if not name.endswith(".yml"):
+            continue
+        data, text, real = _yaml_or_link(fetcher, key, directory, name)
+        boards[name[len("configgen-defaults-"):-len(".yml")]] = (data, _top_key_lines(text), real)
+    rows: list[dict] = []
+    for name, system in systems.items():
+        if not isinstance(system, dict):
+            continue
+        if not system.get("emulators") and not system.get("force"):
+            continue
+        path = system.get("path", name)
+        if not path or str(path).startswith("/"):
+            continue
+        base = defaults.get(name) or {}
+        emulator, core = base.get("emulator"), base.get("core")
+        board_defaults: dict[str, dict] = {}
+        for board, (data, lines, real) in sorted(boards.items()):
+            override = data.get(name) or {}
+            b_emu, b_core = override.get("emulator", emulator), override.get("core", core)
+            if (b_emu, b_core) != (emulator, core) and b_core:
+                board_defaults[board] = _board(b_core, b_emu, _ref(key, real, lines.get(name)))
+        refs = [_ref(key, es_path, es_lines.get(name))]
+        if name in def_lines:
+            refs.append(_ref(key, def_path, def_lines[name]))
+        rows.append({
+            "folder": str(path),
+            "name": name,
+            "extensions": _extensions(system.get("extensions")),
+            "core": str(core) if core else None,
+            "emulator": str(emulator) if emulator else None,
+            "source_ref": "; ".join(refs),
+            "board_defaults": board_defaults,
+            "arcade": system.get("hardware") == "arcade" or None,
+        })
+    return rows
+
+
+# ---------------------------------------------------------------------------
+# Static es_systems.cfg (XML) and EmuELEC's es_systems.json
+
+
+def _first_lines(text: str, pattern: str) -> dict[str, int]:
+    out: dict[str, int] = {}
+    for number, line in enumerate(text.splitlines(), 1):
+        match = re.search(pattern, line)
+        if match:
+            out.setdefault(match.group(1).strip(), number)
+    return out
+
+
+def _all_lines(text: str, pattern: str) -> dict[str, list[int]]:
+    """Every line each name is declared on, in file order."""
+    out: dict[str, list[int]] = {}
+    for number, line in enumerate(text.splitlines(), 1):
+        match = re.search(pattern, line)
+        if match:
+            out.setdefault(match.group(1).strip(), []).append(number)
+    return out
+
+
+def _name_once(row: dict, lines: dict[str, list[int]], seen: dict[str, int]) -> dict:
+    """A system name the list gives twice designates neither: the row keeps
+    its folder and the line of its own occurrence."""
+    name = row["name"]
+    index = seen.get(name, 0)
+    seen[name] = index + 1
+    found = lines.get(name, [])
+    row["line"] = found[index] if index < len(found) else None
+    if len(found) > 1:
+        row["note"] = "; ".join(filter(None, [row.get("note"), f"the list names {len(found)} systems {name}"]))
+        row["name"] = None
+    return row
+
+
+def _pick_default(emulators: list[tuple[str, list[tuple[str, bool]]]], rule: str) -> tuple[str | None, str | None]:
+    """The emulator and core the frontend starts when the user chose none.
+
+    `flagged`: the core marked default, else the first emulator's first
+    core. `first`: the first emulator's first core, whatever is flagged.
+    """
+    if rule == "flagged":
+        for emulator, cores in emulators:
+            for core, default in cores:
+                if default:
+                    return emulator, core
+    for emulator, cores in emulators:
+        if cores:
+            return emulator, cores[0][0]
+        return emulator, None
+    return None, None
+
+
+_LIBRETRO_SO = re.compile(r"-L\s+\S*?[/\\]([\w.+-]+?)_libretro\.(?:so|dll|dylib)")
+
+
+def _libretro_in_command(command: str) -> tuple[str | None, str | None]:
+    match = _LIBRETRO_SO.search(command)
+    return ("retroarch", match.group(1)) if match else (None, None)
+
+
+def _is_arcade(hardware: str | None, platform: str | None) -> bool | None:
+    words = set(re.split(r"[\s,]+", (platform or "").lower()))
+    return (hardware or "").lower() == "arcade" or "arcade" in words or None
+
+
+@register("es-xml")
+def es_xml(spec: dict, fetcher: Fetcher) -> list[dict]:
+    """A static es_systems.cfg. `boards` names the file another build
+    installs instead (dArkOS copies es_systems.cfg.<chipset>): a system whose
+    default differs there gets that board's default."""
+    params = spec["params"]
+    key = params.get("source", "os")
+    rows = _xml_rows(fetcher.text(key, params["file"]), key, params["file"], params)
+    for board, path in sorted((params.get("boards") or {}).items()):
+        other = {r["name"]: r for r in _xml_rows(fetcher.text(key, path), key, path, params)}
+        for row in rows:
+            alt = other.get(row["name"])
+            if alt and alt["core"] and (alt["core"], alt["emulator"]) != (row["core"], row["emulator"]):
+                row.setdefault("board_defaults", {})[board] = _board(
+                    alt["core"], alt["emulator"], alt["source_ref"])
+    return rows
+
+
+_BARE_AMPERSAND = re.compile(r"&(?!(?:[A-Za-z]+|#\d+|#x[0-9A-Fa-f]+);)")
+_COMMENT = re.compile(r"<!--.*?-->", re.DOTALL)
+
+
+def _lenient_xml(text: str) -> str:
+    """The list as EmulationStation's pugixml reads it, lines kept in place.
+
+    pugixml keeps a bare `&` as text (ArkOS ships `2>&1` in a command) and
+    accepts `--` inside a comment (EmuDeck comments out a command holding
+    `--no-gui`). A comment is blanked to its newlines, so a system left in
+    a comment neither parses nor lends its line to a citation.
+    """
+    text = _COMMENT.sub(lambda m: "\n" * m.group(0).count("\n"), text)
+    return _BARE_AMPERSAND.sub("&amp;", text)
+
+
+def _xml_rows(text: str, key: str, path: str, params: dict) -> list[dict]:
+    text = _lenient_xml(text)
+    root = parse_untrusted_xml(text, path)
+    lines = _all_lines(text, r"<name>([^<]+)</name>")
+    seen: dict[str, int] = {}
+    rule = params.get("default", "flagged")
+    rows: list[dict] = []
+    for system in root.iter("system"):
+        name = (system.findtext("name") or "").strip()
+        command = (system.findtext("command") or "").strip()
+        platform = (system.findtext("platform") or "").strip()
+        row = _name_once({"name": name}, lines, seen)
+        if not name or "ignore" in (command, platform):
+            continue
+        folder = _strip_prefix((system.findtext("path") or "").strip(), params["path_prefixes"])
+        if not folder:
+            continue
+        emulators = [
+            (emulator.get("name") or "",
+             [((core.text or "").strip(), core.get("default") == "true")
+              for core in emulator.iter("core") if (core.text or "").strip()])
+            for emulator in system.iter("emulator")
+        ]
+        emulator, core = _pick_default(emulators, rule)
+        if not emulators:
+            # No <emulators>: the command starts the core itself (ArkOS
+            # `retroarch -L .../mess_libretro.so %ROM%`).
+            emulator, core = _libretro_in_command(command)
+        rows.append({
+            "folder": folder,
+            "name": row["name"],
+            "extensions": _extensions(system.findtext("extension")),
+            "core": core or emulator,
+            "emulator": emulator,
+            "source_ref": _ref(key, path, row["line"]),
+            "arcade": _is_arcade(system.findtext("hardware"), platform),
+            "note": row.get("note"),
+        })
+    return rows
+
+
+@register("es-json")
+def es_json(spec: dict, fetcher: Fetcher) -> list[dict]:
+    params = spec["params"]
+    key = params.get("source", "os")
+    path = params["file"]
+    text = fetcher.text(key, path)
+    lines = _all_lines(text, r'^\s{4,6}"name":\s*"([^"]+)"')
+    seen: dict[str, int] = {}
+    rows: list[dict] = []
+    for system in json.loads(text).get("systems", []):
+        name = system.get("name") or ""
+        row = _name_once({"name": name}, lines, seen)
+        if "ignore" in (system.get("command"), system.get("platform")):
+            continue
+        folder = _strip_prefix(system.get("path") or "", params["path_prefixes"])
+        if not name or not folder:
+            continue
+        emulators = [
+            (emulator.get("name") or "",
+             [(c.get("name") or "", bool(c.get("default"))) for c in emulator.get("cores") or []])
+            for emulator in system.get("emulators") or []
+        ]
+        emulator, core = _pick_default(emulators, params.get("default", "flagged"))
+        rows.append({
+            "folder": folder,
+            "name": row["name"],
+            "extensions": _extensions(system.get("extensions")),
+            "core": core or emulator,
+            "emulator": emulator,
+            "source_ref": _ref(key, path, row["line"]),
+            "arcade": _is_arcade(system.get("hardware"), system.get("platform")),
+            "note": row.get("note"),
+        })
+    return rows
+
+
+# ---------------------------------------------------------------------------
+# ROCKNIX family: config/emulators/<system>.conf and the emulators package
+
+
+def _conf_values(text: str) -> tuple[dict[str, str], dict[str, int]]:
+    values: dict[str, str] = {}
+    lines: dict[str, int] = {}
+    for number, line in enumerate(text.splitlines(), 1):
+        match = re.match(r"^\s*(SYSTEM_[A-Z_]+)=(.*)$", line)
+        if not match:
+            continue
+        raw = match.group(2).strip()
+        try:
+            value = " ".join(shlex.split(raw))
+        except ValueError:
+            value = raw.strip('"')
+        values[match.group(1)] = value
+        lines[match.group(1)] = number
+    return values, lines
+
+
+def _package_calls(text: str) -> tuple[list[dict], dict[str, list[int]]]:
+    """add_emu_core and add_es_system calls of makeinstall_target, each
+    with the `case` branches it sits under as (variable, pattern) pairs.
+    An `if` block nests a condition whose truth the build decides: its
+    calls carry it as a branch the default choice does not count as
+    unconditional."""
+    cores: list[dict] = []
+    systems: dict[str, list[int]] = {}
+    stack: list[list[str]] = []
+    inside = False
+    for number, raw in enumerate(text.splitlines(), 1):
+        line = raw.split("#", 1)[0].strip()
+        if raw.startswith("makeinstall_target()"):
+            inside = True
+            continue
+        if not inside or not line:
+            continue
+        if raw.startswith("}"):
+            break
+        match = re.match(r'case\s+"?\$\{?(\w+)\}?"?\s+in', line)
+        if match:
+            stack.append(["case", match.group(1), ""])
+            continue
+        if line == "esac":
+            stack.pop()
+            continue
+        if line.startswith("if ") or line.startswith("[") and "&&" not in line and line.endswith("then"):
+            stack.append(["if", line, ""])
+            continue
+        if line in ("fi",):
+            if stack and stack[-1][0] == "if":
+                stack.pop()
+            continue
+        if line in ("then", "else") or line.startswith("elif "):
+            continue
+        ends = line.endswith(";;")
+        if ends:
+            line = line[:-2].strip()
+        if stack and stack[-1][0] == "case":
+            match = re.match(r"^([^()\s][^()]*)\)\s*(.*)$", line)
+            if match and not line.startswith(("add_", "[", "sed", "cp", "mkdir")):
+                stack[-1][2] = match.group(1).strip()
+                line = match.group(2).strip()
+        words = line.split()
+        branches = [(s[1], s[2]) for s in stack]
+        if words[:1] == ["add_emu_core"] and len(words) >= 4:
+            cores.append({
+                "system": words[1], "emulator": words[2], "core": words[3],
+                "default": len(words) > 4 and words[4] == "true",
+                "branches": branches, "line": number,
+            })
+        elif words[:1] == ["add_es_system"] and len(words) >= 2:
+            systems.setdefault(words[1], []).append(number)
+        if ends and stack and stack[-1][0] == "case":
+            stack[-1][2] = ""
+    return cores, systems
+
+
+def _board_key(branches: list[tuple[str, str]]) -> str | None:
+    """A default chosen under one `case` on the device names that device."""
+    if len(branches) == 1 and branches[0][1] and branches[0][1] != "*":
+        variable, pattern = branches[0]
+        if not variable.startswith("["):
+            return f"{variable}={pattern}"
+    return None
+
+
+@register("elec-conf")
+def elec_conf(spec: dict, fetcher: Fetcher) -> list[dict]:
+    """A system ES lists is one add_es_system writes from its conf file
+    (SYSTEM_PATH, SYSTEM_EXTENSION). The default is the core whose
+    add_emu_core says true: unconditional, else the `*)` branch of a
+    device case, the branches for named devices going to board_defaults."""
+    params = spec["params"]
+    key = params.get("source", "os")
+    package = params["package"]
+    cores, systems = _package_calls(fetcher.text(key, package))
+    if not systems:
+        raise SourceError(f"{package}: no add_es_system call")
+    conf_dir = params["conf_dir"]
+    rows: list[dict] = []
+    for name, lines in systems.items():
+        conf_path = f"{conf_dir}/{name}.conf"
+        conf_text = fetcher.optional_text(key, conf_path)
+        if conf_text is None:
+            continue
+        values, value_lines = _conf_values(conf_text)
+        folder = _strip_prefix(values.get("SYSTEM_PATH", ""), params["path_prefixes"])
+        if not folder:
+            continue
+        mine = [c for c in cores if c["system"] == name]
+        flagged = [c for c in mine if c["default"]]
+        plain = [c for c in flagged if not c["branches"]]
+        star = [c for c in flagged if c["branches"] and c["branches"][-1][1] == "*"]
+        pick = (plain or star or flagged or [None])[0]
+        if pick is None:
+            pick = next((c for c in mine if not c["branches"]), mine[0] if mine else None)
+        board_defaults = {}
+        for c in flagged:
+            board = _board_key(c["branches"])
+            if c is not pick and board and (c["emulator"], c["core"]) != (
+                    pick and pick["emulator"], pick and pick["core"]):
+                board_defaults[board] = _board(c["core"], c["emulator"], _ref(key, package, c["line"]))
+        refs = [
+            _ref(key, conf_path, value_lines.get("SYSTEM_PATH"), value_lines.get("SYSTEM_EXTENSION")),
+            _ref(key, package, *lines),
+        ]
+        if pick:
+            refs.append(_ref(key, package, pick["line"]))
+        note = None
+        if pick and pick["branches"] and not _board_key(pick["branches"]) and pick["branches"][-1][1] != "*":
+            note = "default core chosen under a build condition: " + ", ".join(
+                f"{v} {p}".strip() for v, p in pick["branches"])
+        rows.append({
+            "folder": folder,
+            "name": name,
+            "extensions": _extensions(values.get("SYSTEM_EXTENSION")),
+            "core": pick["core"] if pick else None,
+            "emulator": pick["emulator"] if pick else None,
+            "source_ref": "; ".join(refs),
+            "board_defaults": board_defaults,
+            "arcade": values.get("SYSTEM_HARDWARE") == "arcade" or None,
+            "note": note,
+        })
+    return rows
+
+
+# ---------------------------------------------------------------------------
+# ES-DE (EmuDeck, RetroDECK)
+
+
+_RA_CORE = re.compile(r"%CORE_RETROARCH%[/\\]([\w.+-]+?)_libretro")
+_EMULATOR = re.compile(r"%EMULATOR_([A-Za-z0-9_-]+)%")
+
+
+def _command_core(command: str) -> tuple[str | None, str | None]:
+    match = _RA_CORE.search(command)
+    if match:
+        return "retroarch", match.group(1)
+    match = _EMULATOR.search(command)
+    if match:
+        name = match.group(1).lower()
+        return name, name
+    return None, None
+
+
+def _esde_systems(text: str, path: str) -> Iterator[tuple[str, str, list[str], list[tuple[str, str]]]]:
+    root = parse_untrusted_xml(_lenient_xml(text), path)
+    for system in root.iter("system"):
+        name = (system.findtext("name") or "").strip()
+        rom_path = (system.findtext("path") or "").strip()
+        commands = [((c.get("label") or "").strip(), (c.text or "").strip())
+                    for c in system.findall("command")]
+        yield name, rom_path, _extensions(system.findtext("extension")), commands
+
+
+def _alternative_emulators(text: str, function: str) -> dict[str, tuple[str, int]]:
+    """`ESDE_setEmu '<label>' <system>` calls inside one function."""
+    out: dict[str, tuple[str, int]] = {}
+    inside = False
+    for number, line in enumerate(text.splitlines(), 1):
+        if re.search(rf"\b{re.escape(function)}\b\s*\(\)|function\s+{re.escape(function)}\b", line):
+            inside = True
+            continue
+        if not inside:
+            continue
+        if line.rstrip() == "}":
+            break
+        match = re.search(r"ESDE_setEmu\s+['\"]([^'\"]+)['\"]\s+([\w.-]+)", line)
+        if match:
+            out[match.group(2)] = (match.group(1), number)
+    return out
+
+
+@register("esde")
+def esde(spec: dict, fetcher: Fetcher) -> list[dict]:
+    """Each system of ES-DE's list whose path is %ROMPATH%/<folder>.
+
+    The default emulator is the first <command> (FileData.cpp), unless an
+    alternativeEmulator label is written for the system: an installer that
+    sets one names a command of that system by its label. A custom system
+    file replaces the bundled definition of the same name.
+    """
+    params = spec["params"]
+    systems: dict[str, dict] = {}
+    for item in [params["systems"], *params.get("custom", [])]:
+        key, path = item["source"], item["path"]
+        text = _lenient_xml(fetcher.text(key, path))
+        lines = _first_lines(text, r"<name>([^<]+)</name>")
+        for name, rom_path, extensions, commands in _esde_systems(text, path):
+            folder = _strip_prefix(rom_path, ["%ROMPATH%/", "%ROMPATH%\\"])
+            if not name or not folder:
+                continue
+            systems[name] = {
+                "folder": folder, "extensions": extensions, "commands": commands,
+                "ref": _ref(key, path, lines.get(name)),
+            }
+    alternatives: dict[str, tuple[str, int]] = {}
+    alt = params.get("alternatives")
+    if alt:
+        alternatives = _alternative_emulators(fetcher.text(alt["source"], alt["path"]), alt["function"])
+        if not alternatives:
+            raise SourceError(f"{alt['path']}: no ESDE_setEmu call in {alt['function']}")
+    rows: list[dict] = []
+    for name, system in systems.items():
+        commands = system["commands"]
+        refs = [system["ref"]]
+        chosen = commands[0] if commands else ("", "")
+        note = None
+        if name in alternatives:
+            label, line = alternatives[name]
+            match = next((c for c in commands if c[0] == label), None)
+            refs.append(_ref(alt["source"], alt["path"], line))
+            if match:
+                chosen = match
+            else:
+                note = f"alternative emulator '{label}' names no command of this system"
+        emulator, core = _command_core(chosen[1])
+        extensions = system["extensions"]
+        if "." in extensions:
+            # ES-DE's "." is a file without an extension (SystemData.cpp
+            # getExtension), not a suffix the schema can hold.
+            extensions = [e for e in extensions if e != "."]
+            note = "; ".join(filter(None, [note, "files without an extension are games too ('.' in the list)"]))
+        rows.append({
+            "folder": system["folder"],
+            "name": name,
+            "extensions": extensions,
+            "core": core,
+            "emulator": emulator,
+            "source_ref": "; ".join(refs),
+            "note": note,
+        })
+    return rows
+
+
+# ---------------------------------------------------------------------------
+# Recalbox
+
+
+def _ini_sections(text: str) -> tuple[dict[str, dict[str, str]], dict[str, int]]:
+    sections: dict[str, dict[str, str]] = {}
+    lines: dict[str, int] = {}
+    current: dict[str, str] | None = None
+    name = ""
+    for number, raw in enumerate(text.splitlines(), 1):
+        line = raw.strip()
+        if not line or line.startswith((";", "#")):
+            continue
+        if line.startswith("[") and line.endswith("]"):
+            name = line[1:-1]
+            current = sections.setdefault(name, {})
+            lines.setdefault(name, number)
+        elif current is not None and "=" in line:
+            k, v = line.split("=", 1)
+            current[k.strip()] = v.strip().strip('"')
+            lines.setdefault(f"{name}/{k.strip()}", number)
+    return sections, lines
+
+
+@register("recalbox")
+def recalbox(spec: dict, fetcher: Fetcher) -> list[dict]:
+    """One system.ini per system. The default core is the lowest
+    `priority` (EmulatorManager), and `priority.<board>` changes it on that
+    board. A `.ext!` extension is an archive the frontend hands over
+    unopened."""
+    params = spec["params"]
+    key = params.get("source", "os")
+    rows: list[dict] = []
+    for entry in fetcher.entries(key, params["systems_dir"]):
+        path = f"{entry}/system.ini"
+        text = fetcher.optional_text(key, path)
+        if text is None:
+            continue
+        sections, lines = _ini_sections(text)
+        system = sections.get("system", {})
+        folder = _strip_prefix(system.get("roms.folder", ""), ["%ROOT%/"])
+        if not folder:
+            continue
+        cores = []
+        for section, values in sections.items():
+            parts = section.split(".")
+            if parts[0] != "core" or len(parts) != 2:
+                continue
+            cores.append({
+                "section": section, "emulator": values.get("emulator"), "core": values.get("core"),
+                "priority": int(values.get("priority", "255")),
+                "boards": {k.split(".", 1)[1]: int(v) for k, v in values.items() if k.startswith("priority.")},
+                "extensions": values.get("extensions", "").split(),
+            })
+        if not cores:
+            continue
+        # EmulatorManager walks the cores from the last and keeps a strictly
+        # lower priority: on a tie the last core listed wins.
+        pick = min(reversed(cores), key=lambda c: c["priority"])
+        board_defaults: dict[str, dict] = {}
+        for board in sorted({b for c in cores for b in c["boards"]}):
+            best = min(reversed(cores), key=lambda c: c["boards"].get(board, c["priority"]))
+            if best is not pick:
+                board_defaults[board] = _board(
+                    best["core"], best["emulator"],
+                    _ref(key, path, lines.get(f"{best['section']}/priority.{board}")))
+        extensions = sorted({e for c in cores for e in c["extensions"]})
+        archive = any(e.endswith("!") for e in extensions)
+        rows.append({
+            "folder": folder,
+            "name": system.get("name"),
+            "extensions": [e.rstrip("!") for e in extensions],
+            "core": pick["core"],
+            "emulator": pick["emulator"],
+            "source_ref": "; ".join([
+                _ref(key, path, lines.get("system/roms.folder")),
+                _ref(key, path, lines.get(f"{pick['section']}/priority"),
+                     lines.get(f"{pick['section']}/core")),
+            ]),
+            "board_defaults": board_defaults,
+            "arcade": sections.get("properties", {}).get("type") == "arcade" or None,
+            "note": "an extension ending in ! is an archive the frontend does not open" if archive else None,
+        })
+    return rows
+
+
+# ---------------------------------------------------------------------------
+# RetroPie
+
+
+_MODULE_FIELD = re.compile(r'^rp_module_(id|section|flags)="([^"]*)"', re.MULTILINE)
+_ADD_EMULATOR = re.compile(r'^\s*addEmulator\s+(\S+)\s+("[^"]*"|\S+)\s+("[^"]*"|\S+)')
+_LIST = re.compile(r'(?:local\s+)?(\w+)=\(([^)]*)\)')
+_LOOP = re.compile(r'for (\w+) in (.+?); do')
+
+
+def _module_valid(flags: list[str], platform: set[str]) -> bool:
+    """packages.sh rp_registerModule: flags read in order, `!all` drops
+    the module, a platform flag keeps it, `!<platform flag>` drops it."""
+    valid = True
+    for flag in flags:
+        if flag == "!all":
+            valid = False
+        elif flag in platform:
+            valid = True
+        elif flag.startswith("!") and flag[1:] in platform:
+            valid = False
+    return valid
+
+
+def _emulator_calls(text: str) -> Iterator[tuple[int, str, str, list[str]]]:
+    """(line, default flag, emulator id, systems) of each addEmulator call,
+    a system variable expanded through the loop or array that sets it."""
+    lines = text.splitlines()
+    arrays = {m.group(1): m.group(2).replace('"', "").split() for m in _LIST.finditer(text)}
+    loops: list[tuple[int, str, list[str]]] = []
+    for index, line in enumerate(lines):
+        match = _LOOP.search(line)
+        if match:
+            items = match.group(2).strip()
+            array = re.fullmatch(r'"?\$\{(\w+)\[@\]\}"?', items)
+            loops.append((index, match.group(1),
+                          arrays.get(array.group(1), []) if array else items.replace('"', "").split()))
+    for index, line in enumerate(lines):
+        match = _ADD_EMULATOR.match(line)
+        if not match or "function addEmulator" in line:
+            continue
+        flag, emu, system = match.group(1), match.group(2).strip('"'), match.group(3).strip('"')
+        systems = [system]
+        var = re.fullmatch(r"\$\{?(\w+)\}?", system)
+        if var:
+            found = [items for i, name, items in loops if i < index and name == var.group(1)]
+            if found:
+                systems = found[-1]
+            elif var.group(1) in arrays:
+                systems = arrays[var.group(1)]
+            else:
+                continue
+        yield index + 1, flag, emu, systems
+
+
+@register("retropie")
+def retropie(spec: dict, fetcher: Fetcher) -> list[dict]:
+    """Folders are `<system>` under the roms root (helpers.sh addSystem),
+    with `<system>_exts` from platforms.cfg. The default is set once, by
+    the first installed module whose addEmulator passes 1: a basic install
+    installs the core then the main section in registration order
+    (setup.sh basic_install_setup, packages.sh rp_registerAllModules). A
+    system no basic-install module serves gets its folder from an optional
+    module, so the first opt, then exp, module stands as its default."""
+    params = spec["params"]
+    key = params.get("source", "os")
+    cfg_path = params["platforms_cfg"]
+    cfg = fetcher.text(key, cfg_path)
+    exts: dict[str, tuple[str, int]] = {}
+    for number, line in enumerate(cfg.splitlines(), 1):
+        match = re.match(r'^([\w-]+)_exts="([^"]*)"', line)
+        if match:
+            exts[match.group(1)] = (match.group(2), number)
+    platform = set(params["platform_flags"])
+    modules: list[dict] = []
+    for kind in params["module_dirs"]:
+        for path in fetcher.tree(key, f"scriptmodules/{kind}/"):
+            if not path.endswith(".sh") or path.count("/") != 2:
+                continue
+            text = fetcher.text(key, path)
+            fields = dict(_MODULE_FIELD.findall(text))
+            if not _module_valid(fields.get("flags", "").split(), platform):
+                continue
+            modules.append({"path": path, "id": fields.get("id", ""),
+                            "section": fields.get("section", ""), "text": text})
+    order = [m for section in params["install_sections"] for m in modules if m["section"] == section]
+    defaults: dict[str, tuple[str, str, int]] = {}
+    for module in order:
+        for line, flag, emu, systems in _emulator_calls(module["text"]):
+            if flag != "1":
+                continue
+            emu = emu.replace("${md_id}", module["id"]).replace("$md_id", module["id"])
+            for system in systems:
+                defaults.setdefault(system, (emu, module["path"], line))
+    rows: list[dict] = []
+    for system, (raw, line) in sorted(exts.items()):
+        refs = [_ref(key, cfg_path, line)]
+        core = None
+        if system in defaults:
+            core, path, call = defaults[system]
+            refs.append(_ref(key, path, call))
+        emulator = "retroarch" if core and core.startswith("lr-") else core
+        rows.append({
+            "folder": system,
+            "extensions": _extensions(raw),
+            "core": core,
+            "emulator": emulator,
+            "source_ref": "; ".join(refs),
+        })
+    return rows

scripts/scraper/romset_dat_importer.py

@@ -200,20 +200,22 @@ def _scan_listxml(
     and those holding one of *targets*.
 
     The document is 311 MB for MAME 0.289, so it is never held whole: the
-    reader keeps a sliding window just large enough to close the element it
-    is in the middle of.
+    window keeps the element still open from its start tag, and grows until
+    that element closes, whatever its size.
     """
     found: dict[str, tuple[str | None, list[dict]]] = {}
     buffer = b""
     while chunk := stream.read(LISTXML_CHUNK):
         buffer += chunk
         position = 0
+        keep_from = None
         while True:
             match = _MACHINE.search(buffer, position)
             if not match:
                 break
             end = buffer.find(b"</machine>", match.end())
             if end == -1:
+                keep_from = match.start()
                 break
             name = match.group(1).decode()
             wanted = selector(name)
@@ -223,7 +225,12 @@ def _scan_listxml(
                     parent = _ROMOF.search(match.group(2).decode())
                     found[name] = (parent.group(1) if parent else None, roms)
             position = end + len(b"</machine>")
-        buffer = buffer[position:] if position else buffer[-(2 * LISTXML_CHUNK) :]
+        if keep_from is None:
+            # No element open: only a start tag cut by the read can follow.
+            start = buffer.rfind(b"<machine", position)
+            tail = max(position, len(buffer) - len(b"<machine"))
+            keep_from = start if start != -1 else tail
+        buffer = buffer[keep_from:]
     return found
 
 

scripts/sitediagrams.py

@@ -882,8 +882,9 @@ GLANCE_HEADING = "## Options at a glance"
 GLANCE_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = (
     ("Where and for what", ("--platform", "--dest", "--target", "--every-board", "--save-dir",
                             "--romm-firmware", "--standalone-copies", "--add-app")),
-    ("Fewer files", ("--system", "--core", "--emulator", "--region", "--no-game-data",
-                     "--declared-only", "--installed-cores", "--cores-dir")),
+    ("Fewer files", ("--for-games", "--required-only", "--system", "--core", "--emulator",
+                     "--region", "--no-game-data", "--declared-only", "--installed-cores",
+                     "--cores-dir")),
     ("Look first", ("--check", "--dry-run", "--list-platforms", "--list-targets",
                     "--list-systems", "--list-cores", "--list-emulators")),
     ("From files you have", ("--from", "--from-pack", "--manifest", "--offline", "--restore")),

scripts/siterender.py

@@ -860,7 +860,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "apple-iie": "Apple IIe",
     "apple-ios": "Apple iOS",
     "apple2": "Apple II",
-    "apple2gs": "Apple IIGS",
     "arcade": "Arcade games",
     "arcade-laserdisc": "Laserdisc arcade games",
     "arcadia-systems": "Arcadia Systems Multi Select",
@@ -869,7 +868,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "aristocrat-mk6": "Aristocrat MK6",
     "aristocrat-mk7": "Aristocrat MK7",
     "arm": "ARM",
-    "astrocde": "Bally Astrocade",
     "atari-falcon": "Atari Falcon",
     "atari-jaguarcd": "Atari Jaguar CD",
     "atari-ste": "Atari STE",
@@ -920,10 +918,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "commodore-cdtv": "Commodore CDTV",
     "commodore-cubo": "CD Express Cubo CD32",
     "coolbar": "Coolbar feature phones",
-    "cps1": "Capcom CPS-1",
-    "cps2": "Capcom CPS-2",
-    "cps3": "Capcom CPS-3",
-    "crvision": "VTech CreatiVision",
     "cstrike": "Counter-Strike",
     "ctf": "Quake II: Capture the Flag",
     "czero": "Counter-Strike: Condition Zero",
@@ -1073,7 +1067,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "nintendo-nss": "Nintendo Super System",
     "nintendo-playch10": "Nintendo PlayChoice-10",
     "nintendo-sfcbox": "Nintendo Super Famicom Box",
-    "nintendo-sgb": "Nintendo Super Game Boy",
     "nintendo-virtualboy": "Nintendo Virtual Boy",
     "nintendo-wii-arcade": "Nintendo Wii arcade boards",
     "nmk16": "NMK16",
@@ -1092,7 +1085,6 @@ SYSTEM_NAMES: dict[str, str] = {
     "pce-tourvision": "TourVision",
     "pdark": "Perfect Dark",
     "pegasus": "Aamber Pegasus",
-    "pgm": "IGS PolyGame Master",
     "philips-p2000t": "Philips P2000T",
     "philips-vg5k": "Philips VG 5000",
     "picocomputer-6502": "Picocomputer 6502 programs",
@@ -1216,6 +1208,7 @@ SYSTEM_NAMES: dict[str, str] = {
     "tomb-raider-1": "Tomb Raider",
     "tomb-raider-2": "Tomb Raider II",
     "transport-tycoon-deluxe": "Transport Tycoon Deluxe",
+    "tvgames": "Plug-and-play TV games",
     "tyrian": "Tyrian",
     "ur-quan-masters": "Star Control II",
     "urban-terror": "Urban Terror",

scripts/validate_schemas.py

@@ -623,6 +623,109 @@ def _measurement_errors(root: Path | None = None) -> list[str]:
     return out
 
 
+def _defs_validator(name: str, definition: str) -> Draft202012Validator:
+    """A validator for one $defs entry of a schema file."""
+    schema = _load_json(SCHEMAS / name)
+    Draft202012Validator.check_schema(schema)
+    return Draft202012Validator(
+        {**schema, "$ref": f"#/$defs/{definition}"}, format_checker=FormatChecker()
+    )
+
+
+def _romfolder_errors() -> list[str]:
+    """Every scraped setup against its schema, and every source_ref against
+    the sources the file declares: a ref naming no declared repository can
+    never be read back at a pin."""
+    directory = ROOT / "platforms" / "romfolders"
+    if not directory.is_dir():
+        return []
+    validator = _validator("romfolders.schema.json")
+    out: list[str] = []
+    for path in sorted(directory.glob("*.yml")):
+        label = str(path.relative_to(ROOT))
+        with path.open(encoding="utf-8") as handle:
+            data = yaml_load(handle)
+        errors = _errors(validator, data, label)
+        out.extend(errors)
+        if errors:
+            continue
+        declared = set(data["sources"])
+        refs = [data["roms_root"]["source_ref"]]
+        refs += [data["case_ref"]] if data.get("case_ref") else []
+        refs += [r["source_ref"] for key in ("skip", "hidden", "media") for r in data.get(key, [])]
+        for row in data["folders"]:
+            refs.append(row["source_ref"])
+            refs += [b["source_ref"] for b in (row.get("board_defaults") or {}).values()]
+        for ref in refs:
+            for part in ref.split("; "):
+                key = part.split(" ", 1)[0]
+                if key not in declared:
+                    out.append(f"{label}: source_ref {part!r} names undeclared source {key!r}")
+    return out
+
+
+# Source data of the games-on-card mode: file pattern -> schema.
+_GAMES_SOURCES = (
+    ("games/catalogs/*/*.json", "game-catalog.schema.json"),
+    ("games/arcade/*.json", "arcade-parents.schema.json"),
+    ("games/headers.yml", "game-headers.schema.json"),
+    ("games/regions.yml", "game-regions.schema.json"),
+    ("games/extras.yml", "game-extras.schema.json"),
+)
+
+
+def _games_source_errors() -> list[str]:
+    out: list[str] = []
+    for pattern, schema in _GAMES_SOURCES:
+        paths = sorted(ROOT.glob(pattern))
+        if not paths:
+            continue
+        validator = _validator(schema)
+        for path in paths:
+            with path.open(encoding="utf-8") as handle:
+                data = json.load(handle) if path.suffix == ".json" else yaml_load(handle)
+            out.extend(_errors(validator, data, str(path.relative_to(ROOT))))
+    return out
+
+
+def _games_index_errors() -> list[str]:
+    """install/games/: each document against its kind, and the index
+    against the files it lists (size and sha256)."""
+    import hashlib
+
+    base = ROOT / "install" / "games"
+    index_path = base / "index.json"
+    if not index_path.exists():
+        return []
+    out: list[str] = []
+    index = _load_json(index_path)
+    out.extend(_errors(_defs_validator("games-index.schema.json", "index"), index,
+                       str(index_path.relative_to(ROOT))))
+    kinds = {
+        "setups": _defs_validator("games-index.schema.json", "setup"),
+        "fallback": _defs_validator("games-index.schema.json", "fallback"),
+        "systems": _defs_validator("games-index.schema.json", "system"),
+        "arcade": _validator("arcade-parents.schema.json"),
+    }
+    listed: set[Path] = set()
+    for kind, validator in kinds.items():
+        for entry in (index.get(kind) or {}).values():
+            path = base / entry["file"]
+            listed.add(path)
+            label = str(path.relative_to(ROOT))
+            if not path.exists():
+                out.append(f"{label}: listed in index.json, absent")
+                continue
+            data = path.read_bytes()
+            if len(data) != entry["size"] or hashlib.sha256(data).hexdigest() != entry["sha256"]:
+                out.append(f"{label}: size or sha256 differs from index.json")
+            out.extend(_errors(validator, json.loads(data), label))
+    for path in sorted(base.rglob("*.json")):
+        if path != index_path and path not in listed:
+            out.append(f"{path.relative_to(ROOT)}: not listed in index.json")
+    return out
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__)
     parser.add_argument(
@@ -647,6 +750,8 @@ def main() -> int:
     errors.extend(_registry_contract())
     errors.extend(_event_schema_errors())
     errors.extend(_measurement_errors())
+    errors.extend(_romfolder_errors())
+    errors.extend(_games_source_errors())
 
     if not args.source_only:
         database_path = ROOT / "database.json"
@@ -702,6 +807,7 @@ def main() -> int:
             )
         )
 
+        errors.extend(_games_index_errors())
         errors.extend(_validate_pack_manifests(ROOT / "dist"))
 
     if errors: