feat: read RetroPie launchers from scriptmodules

The scraper walks each package's functions the way RetroPie runs them in one shell (assignments, arrays, for loops, brace expansion, $md_id, calls into other packages) and records every link, copy, move and setting that names $biosdir, the addEmulator flags (sdltrs -romfile3), the BIOS read outside $biosdir (basilisk --rom, gngeopi -B) and the launch scripts that exit without their file (fs-uae kick13.rom, reicast dc_boot.bin).

Each package also gets a package record with its rp_module_repo and the systems it registers, which maps a package to the build it compiles. The schema gains package, repo, systems and the moves and package kinds. platforms/retropie.yml is regenerated at 46128cd2: only the launchers section is new.

scripts/scraper/retropie_launchers.py

@@ -0,0 +1,995 @@
+"""RetroPie's launcher layer, read from the scriptmodules.
+
+RetroPie publishes no BIOS list, but its package scripts decide where a file
+dropped in ~/RetroPie/BIOS ends up: px68k links five names into ~/.keropi,
+redream links dc_boot.bin under another name, sdltrs passes one ROM per
+machine on its command line, the fs-uae launch script refuses to start
+without kick13.rom. Each package also says which project it builds
+(`rp_module_repo`) and which systems it registers (`addSystem`), which is
+what ties a package to the emulator profile describing that build.
+
+The bash is read as text, never run. A small evaluator follows what the
+scripts do between a function's braces: local assignments, arrays, `for`
+loops, brace expansion, `$md_id` and calls into other packages' functions,
+the way RetroPie runs them in one shell. Every line of a package function
+that names the BIOS directory must match a known construct, else the scrape
+stops with LauncherPatternMissing: a use nobody classified would otherwise
+drop out of the platform file without a word.
+"""
+
+from __future__ import annotations
+
+import re
+import shlex
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from itertools import product
+from pathlib import PurePosixPath
+from typing import Any
+
+SCRIPTMODULES = "scriptmodules"
+SECTIONS = ("emulators", "libretrocores", "ports")
+PACKAGES_SCRIPT = "retropie_packages.sh"
+
+BIOSDIR = "$biosdir"
+# The package whose config every lr- core runs under.
+LIBRETRO_FRONTEND = "retroarch"
+# The launch scripts copied next to an emulator spell the directory out.
+BIOS_SPELLED = "RetroPie/BIOS"
+
+# Commands whose $biosdir argument creates, owns or deletes a path and hands
+# nothing to an emulator.
+HOUSEKEEPING = frozenset({"mkUserDir", "chown", "chmod", "rm", "mkdir", "iniConfig"})
+
+# Arguments that name a BIOS file or directory outside $biosdir, per package,
+# as the emulator documents the flag. Each must still be in the scriptmodule.
+OUTSIDE_BIOS: dict[str, tuple[str, ...]] = {
+    "basilisk": ("--rom",),  # BasiliskII --rom <ROM file>
+    "gngeopi": ("-B",),  # gngeo -B/--biospath <directory of neogeo.zip>
+}
+
+_FUNCTION = re.compile(r"^function\s+([\w.+-]+)\s*\(\)\s*\{\s*$")
+_ASSIGN = re.compile(r"^(?:local\s+)?([A-Za-z_]\w*)=(.*)$")
+_DECLARE = re.compile(r"^local\s+([A-Za-z_]\w*(?:\s+[A-Za-z_]\w*)*)\s*$")
+_FOR = re.compile(r"^for\s+([A-Za-z_]\w*)\s+in\s+(.*?)\s*(?:;\s*do)?\s*$")
+_HEREDOC = re.compile(r"(?<!<)<<-?(?!<)\s*['\"]?(\w+)['\"]?")
+_VAR = re.compile(r"\$\{([A-Za-z_]\w*)(\[@\])?\}|\$([A-Za-z_]\w*|\d)")
+_SUBST = re.compile(r"\$\(([\w.+-]+(?:\$\{md_id\}|\$md_id)?)\)")
+_MD_ID_TEST = re.compile(r'^!?\s*\[\[\s*"\$md_id"\s*(==|!=)\s*"?([\w.+-]+)"?\s*\]\]$')
+_MODULE_FIELD = re.compile(r'^rp_module_(id|repo)="([^"]*)"', re.MULTILINE)
+_EXTENSION = re.compile(r"\.[A-Za-z0-9]{1,5}$")
+
+
+class LauncherPatternMissing(ValueError):
+    """A construct an extractor reads is absent from the scriptmodules."""
+
+    def __init__(self, path: str, what: str) -> None:
+        super().__init__(f"{path}: {what}")
+
+
+@dataclass
+class Launcher:
+    """One thing a package hands an emulator, or the package itself."""
+
+    emulator: str
+    build: str
+    kind: str
+    names: list[str]
+    source_ref: str
+    package: str | None = None
+    repo: str | None = None
+    systems: list[str] | None = None
+    slot: str | None = None
+    case_insensitive: bool | None = None
+    search_dir: str | None = None
+    passes_to: str | None = None
+    fails_when_missing: bool | str | None = None
+    depends_on_option: str | None = None
+    lines: list[tuple[str, int]] = field(default_factory=list, repr=False)
+
+    def entry(self) -> dict[str, Any]:
+        """The platform YAML form, in a fixed key order, unset keys left out."""
+        order = (
+            "emulator", "package", "repo", "systems", "build", "kind", "slot",
+            "names", "case_insensitive", "search_dir", "passes_to",
+            "fails_when_missing", "depends_on_option", "source_ref",
+        )
+        return {key: getattr(self, key) for key in order if getattr(self, key) is not None}
+
+
+@dataclass
+class Module:
+    """One package script: its id, section, lines and functions."""
+
+    path: str
+    section: str
+    lines: list[str]
+    id: str
+    id_line: int
+    repo: str | None
+    repo_line: int | None
+    functions: dict[str, tuple[int, int]] = field(default_factory=dict)
+
+    @property
+    def build(self) -> str:
+        libretro = self.section == "libretrocores" or self.id == LIBRETRO_FRONTEND
+        return "libretro" if libretro else "standalone"
+
+
+@dataclass
+class Event:
+    """A statement reached while walking a package's functions."""
+
+    path: str
+    line: int
+    words: list[list[str]]
+    raw: str
+    conditions: list[str]
+    heredoc: bool
+    opener: list[list[str]] = field(default_factory=list)
+
+
+def _logical_lines(
+    lines: list[str], start: int, end: int
+) -> Iterator[tuple[int, str, str | None]]:
+    """(line number, text, heredoc opener or None) for lines start..end, continuations joined."""
+    tag: str | None = None
+    opener = ""
+    number = start
+    while number <= end:
+        text = lines[number - 1]
+        first = number
+        if tag is not None:
+            if text.strip() == tag:
+                tag = None
+            else:
+                yield first, text, opener
+            number += 1
+            continue
+        while text.rstrip().endswith("\\") and number < end:
+            number += 1
+            text = text.rstrip()[:-1] + " " + lines[number - 1].strip()
+        opened = _HEREDOC.search(text)
+        if opened is not None:
+            tag = opened.group(1)
+            opener = text.strip()
+        yield first, text, None
+        number += 1
+
+
+def _brace(word: str) -> list[str]:
+    """Bash brace expansion of comma lists, outside variable references."""
+    found = re.search(r"(?<!\$)\{([^{}$]*,[^{}$]*)\}", word)
+    if found is None:
+        return [word]
+    head, tail = word[: found.start()], word[found.end():]
+    return [
+        expanded
+        for option in found.group(1).split(",")
+        for expanded in _brace(head + option + tail)
+    ]
+
+
+def _split(text: str) -> list[str]:
+    """Shell words with quotes removed, a line bash itself would accept."""
+    try:
+        return shlex.split(text, comments=True)
+    except ValueError:
+        return text.split()
+
+
+class Scripts:
+    """Every package script, with the functions RetroPie loads into one shell."""
+
+    def __init__(self, files: dict[str, str]) -> None:
+        self.files = files
+        self.modules: dict[str, Module] = {}
+        self.owner: dict[str, Module] = {}
+        self._walked: dict[str, list[Event]] = {}
+        for path in sorted(files):
+            parts = PurePosixPath(path).parts
+            is_package = (
+                len(parts) == 3
+                and parts[0] == SCRIPTMODULES
+                and parts[1] in SECTIONS
+                and parts[2].endswith(".sh")
+            )
+            if is_package:
+                module = self._parse(path, parts[1], files[path])
+                if module is not None:
+                    self.modules[module.id] = module
+        if not self.modules:
+            raise LauncherPatternMissing(SCRIPTMODULES, "no package script declares rp_module_id")
+        for module in self.modules.values():
+            for name in module.functions:
+                self.owner[name] = module
+
+    @staticmethod
+    def _parse(path: str, section: str, text: str) -> Module | None:
+        lines = text.splitlines()
+        fields: dict[str, tuple[str, int]] = {}
+        for found in _MODULE_FIELD.finditer(text):
+            line = text.count("\n", 0, found.start()) + 1
+            fields.setdefault(found.group(1), (found.group(2), line))
+        if "id" not in fields:
+            return None
+        repo = fields.get("repo")
+        module = Module(
+            path=path, section=section, lines=lines,
+            id=fields["id"][0], id_line=fields["id"][1],
+            repo=repo[0] if repo else None, repo_line=repo[1] if repo else None,
+        )
+        opened: tuple[str, int] | None = None
+        tag: str | None = None
+        for number, line in enumerate(lines, 1):
+            # A generated script may define functions of its own.
+            if tag is not None:
+                if line.strip() == tag:
+                    tag = None
+                continue
+            heredoc = _HEREDOC.search(line)
+            if heredoc is not None:
+                tag = heredoc.group(1)
+            header = _FUNCTION.match(line)
+            if header is not None:
+                opened = (header.group(1), number)
+            elif opened is not None and line.rstrip() == "}":
+                module.functions[opened[0]] = (opened[1], number)
+                opened = None
+        if opened is not None:
+            raise LauncherPatternMissing(path, f"function {opened[0]} has no closing brace")
+        return module
+
+    def echo_value(self, name: str) -> str | None:
+        """What a function whose whole body is one `echo "X"` prints."""
+        module = self.owner.get(name)
+        if module is None:
+            return None
+        start, end = module.functions[name]
+        body = [line.strip() for line in module.lines[start:end - 1] if line.strip()]
+        if len(body) == 1:
+            words = _split(body[0])
+            if len(words) == 2 and words[0] == "echo":
+                return words[1]
+        return None
+
+    def walk(self, package: Module) -> list[Event]:
+        """Every statement of a package's functions, as RetroPie runs them for it."""
+        if package.id not in self._walked:
+            events: dict[tuple[str, int], Event] = {}
+            for name in package.functions:
+                self._walk(package, name, {}, [], set(), events)
+            self._walked[package.id] = sorted(
+                events.values(), key=lambda e: (e.path != package.path, e.path, e.line)
+            )
+        return self._walked[package.id]
+
+    def _walk(
+        self,
+        package: Module,
+        name: str,
+        args: dict[str, list[str]],
+        conditions: list[str],
+        stack: set[str],
+        events: dict[tuple[str, int], Event],
+    ) -> None:
+        if name in stack:
+            return
+        module = self.owner[name]
+        start, end = module.functions[name]
+        env: dict[str, list[str]] = dict(args)
+        loops: list[tuple[str | None, list[str] | None]] = []
+        branches: list[tuple[str, bool | None]] = []
+        array: tuple[str, list[str]] | None = None
+        for number, text, opener in _logical_lines(module.lines, start + 1, end - 1):
+            stripped = text.strip()
+            if array is not None:
+                body = re.sub(r"\s+#.*$", "", stripped)
+                closed = body.endswith(")")
+                array[1].append(body[:-1] if closed else body)
+                if closed:
+                    words = _split(" ".join(array[1]))
+                    env[array[0]] = [v for w in words for v in self.expand(w, package, env)]
+                    array = None
+                continue
+            if opener is not None:
+                if self._live(branches):
+                    words = [self.expand(w, package, env) for w in stripped.split()]
+                    events.setdefault((module.path, number), Event(
+                        module.path, number, words, stripped,
+                        conditions + self._shown(branches, package, env), True,
+                        [self.expand(w, package, env) for w in _split(opener)],
+                    ))
+                continue
+            if not stripped or stripped.startswith("#"):
+                continue
+            if self._branch(stripped, package, env, branches):
+                continue
+            if not self._live(branches):
+                continue
+            loop = _FOR.match(stripped)
+            if loop is not None:
+                values = [v for w in _split(loop.group(2)) for v in self.expand(w, package, env)]
+                loops.append((loop.group(1), env.get(loop.group(1))))
+                env[loop.group(1)] = values
+                continue
+            if re.match(r"^(while|until)\b|^for\s*\(\(", stripped):
+                loops.append((None, None))
+                continue
+            if stripped == "done" or stripped.startswith("done ") or stripped.startswith("done;"):
+                if loops:
+                    var, previous = loops.pop()
+                    if var is not None:
+                        if previous is None:
+                            env.pop(var, None)
+                        else:
+                            env[var] = previous
+                continue
+            if stripped in ("do", "then", "else", "fi", "esac", "{", "}"):
+                continue
+            guard = re.match(r"^\[\[\s*(.*?)\s*\]\]\s*&&\s*(.*)$", stripped)
+            shown = conditions + self._shown(branches, package, env)
+            if guard is not None:
+                verdict = self._md_id_test(f"[[ {guard.group(1)} ]]", package)
+                if verdict is False or "$md_mode" in guard.group(1) or "$mode" in guard.group(1):
+                    continue
+                if verdict is None:
+                    shown = shown + [self._render(guard.group(1), package, env)]
+                stripped = guard.group(2)
+            opens = re.match(r"^(?:local\s+)?([A-Za-z_]\w*)=\(([^)]*)$", stripped)
+            if opens is not None:
+                array = (opens.group(1), [opens.group(2)])
+                continue
+            if self._bind(stripped, package, env):
+                continue
+            words = [self.expand(w, package, env) for w in _split(stripped)]
+            # The first walk to reach a line is the one with its caller's arguments.
+            events.setdefault(
+                (module.path, number), Event(module.path, number, words, stripped, shown, False)
+            )
+            callee = words[0][0] if words and words[0] else None
+            if callee in self.owner and callee != name:
+                call_args = {str(i): w for i, w in enumerate(words[1:], 1)}
+                self._walk(package, callee, call_args, shown, stack | {name}, events)
+
+    def _branch(
+        self, stripped: str, package: Module, env: dict[str, list[str]],
+        branches: list[tuple[str, bool | None]],
+    ) -> bool:
+        """Track if/elif/else/fi, and say whether the line was one of them."""
+        opened = re.match(r"^(if|elif)\s+(.*?)\s*(?:;\s*then)?\s*$", stripped)
+        if opened is not None:
+            condition = opened.group(2)
+            verdict = self._md_id_test(condition, package)
+            if "$md_mode" in condition or "$mode" in condition:
+                verdict = True if "install" in condition else None
+            if opened.group(1) == "if":
+                branches.append((condition, verdict))
+            elif branches:
+                previous, previous_verdict = branches[-1]
+                if previous_verdict is True:
+                    verdict = False
+                branches[-1] = (condition, verdict)
+            return True
+        if stripped == "else" and branches:
+            condition, verdict = branches[-1]
+            branches[-1] = (f"not ({condition})", None if verdict is None else not verdict)
+            return True
+        if stripped == "fi" or stripped.startswith("fi "):
+            if branches:
+                branches.pop()
+            return True
+        return False
+
+    @staticmethod
+    def _live(branches: list[tuple[str, bool | None]]) -> bool:
+        return all(verdict is not False for _c, verdict in branches)
+
+    def _shown(
+        self, branches: list[tuple[str, bool | None]], package: Module, env: dict[str, list[str]]
+    ) -> list[str]:
+        return [
+            self._render(condition, package, env)
+            for condition, verdict in branches if verdict is None
+        ]
+
+    def _render(self, condition: str, package: Module, env: dict[str, list[str]]) -> str:
+        """A condition with its variables filled in when each takes one value."""
+        values = self.expand(condition, package, env)
+        return values[0] if len(values) == 1 else condition
+
+    @staticmethod
+    def _md_id_test(condition: str, package: Module) -> bool | None:
+        test = _MD_ID_TEST.match(condition.strip())
+        if test is None:
+            return None
+        equal = package.id == test.group(2)
+        return equal if test.group(1) == "==" else not equal
+
+    def _bind(self, stripped: str, package: Module, env: dict[str, list[str]]) -> bool:
+        """Record a variable assignment, and say whether the line was one."""
+        if _DECLARE.match(stripped):
+            return True
+        assign = _ASSIGN.match(stripped)
+        if assign is None:
+            return False
+        name, value = assign.group(1), assign.group(2).strip()
+        if value.startswith("("):
+            closing = value.rfind(")")
+            inner = value[1:closing] if closing > 0 else value[1:]
+            env[name] = [v for w in _split(inner) for v in self.expand(w, package, env)]
+            return True
+        words = _split(value) or [""]
+        if "$(" in words[0] and _SUBST.fullmatch(words[0]) is None:
+            # A command's output: the variable stays a reference.
+            env.pop(name, None)
+            return True
+        expanded = self.expand(words[0], package, env)
+        if any("$(" in v for v in expanded):
+            env.pop(name, None)
+        else:
+            env[name] = expanded
+        return True
+
+    def expand(self, word: str, package: Module, env: dict[str, list[str]]) -> list[str]:
+        """Every value a word takes: braces, $md_id, bound variables, $(echo functions)."""
+        word = re.sub(r"\$\{md_id\}|\$md_id\b", package.id, word)
+
+        def substitute(found: re.Match[str]) -> str:
+            printed = self.echo_value(found.group(1))
+            return printed if printed is not None else found.group(0)
+
+        word = _SUBST.sub(substitute, word)
+        results: list[str] = []
+        for braced in _brace(word):
+            names = [m.group(1) or m.group(3) for m in _VAR.finditer(braced)]
+            bound = [n for n in names if n in env]
+            if not bound:
+                results.append(braced)
+                continue
+            for values in product(*(env[n] for n in bound)):
+                chosen = dict(zip(bound, values))
+
+                def replace(found: re.Match[str]) -> str:
+                    key = found.group(1) or found.group(3)
+                    return chosen.get(key, found.group(0))
+
+                results.append(_VAR.sub(replace, braced))
+        return results
+
+
+def _mentions_bios(text: str) -> bool:
+    return BIOSDIR in text or "${biosdir}" in text or BIOS_SPELLED in text
+
+
+def _is_file(path: str) -> bool:
+    return bool(_EXTENSION.search(PurePosixPath(path).name)) and "*" not in path
+
+
+def _split_path(path: str) -> tuple[str, str]:
+    pure = PurePosixPath(path.rstrip("/"))
+    return str(pure.parent), pure.name
+
+
+class PackageReader:
+    """Launcher records of one package, from the statements its functions run."""
+
+    def __init__(self, scripts: Scripts, package: Module) -> None:
+        self.scripts = scripts
+        self.package = package
+        self.records: list[Launcher] = []
+        self.installed_as: dict[str, str] = {}
+
+    def record(
+        self, kind: str, names: list[str], lines: list[tuple[str, int]], **fields: Any
+    ) -> None:
+        self.records.append(Launcher(
+            emulator=fields.pop("emulator", self.package.id),
+            package=self.package.id,
+            build=self.package.build,
+            kind=kind,
+            names=names,
+            source_ref=source_ref(lines),
+            lines=list(lines),
+            **fields,
+        ))
+
+    def final(self, path: str) -> str:
+        """The file a scratch config ends up as, from the copyDefaultConfig that installs it."""
+        return self.installed_as.get(path, path)
+
+    def read(self) -> list[Launcher]:
+        events = self.scripts.walk(self.package)
+        for event in events:
+            if not event.heredoc and event.words and event.words[0] == ["copyDefaultConfig"]:
+                if len(event.words) >= 3 and len(event.words[1]) == 1 and len(event.words[2]) == 1:
+                    self.installed_as.setdefault(event.words[1][0], event.words[2][0])
+        links: list[Event] = []
+        for event in events:
+            if not event.words or not event.words[0]:
+                continue
+            if links and not self._same_link_run(links[-1], event):
+                self._links(links)
+                links = []
+            if self._is_bios_link(event):
+                links.append(event)
+                continue
+            self._classify(event)
+        if links:
+            self._links(links)
+        self._outside_bios(events)
+        return self.records
+
+    @staticmethod
+    def _depends(event: Event) -> str | None:
+        return "; ".join(event.conditions) if event.conditions else None
+
+    def _classify(self, event: Event) -> None:
+        command = event.words[0][0]
+        if event.heredoc:
+            if _mentions_bios(event.raw):
+                self._heredoc_setting(event)
+            return
+        if command == "addEmulator":
+            self._add_emulator(event)
+            return
+        if not _mentions_bios(event.raw):
+            return
+        if command in HOUSEKEEPING:
+            return
+        handlers = {
+            "cp": self._copy,
+            "runCmd": self._run_cmd,
+            "unzip": self._unzip,
+            "mv": self._move,
+            "moveConfigFile": self._move,
+            "moveConfigDir": self._move_dir,
+            "iniSet": self._ini_set,
+            "copyDefaultConfig": self._written,
+            "cat": self._written,
+            "echo": self._written,
+            "cmake": self._build_flag,
+        }
+        handler = handlers.get(command)
+        if handler is None:
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}", f"unrecognised use of the BIOS directory: {event.raw}"
+            )
+        handler(event)
+
+    def _is_bios_link(self, event: Event) -> bool:
+        words = event.words
+        if words[0][0] != "ln" or not _mentions_bios(event.raw):
+            return False
+        paths = [w for w in words[1:] if not w[0].startswith("-")]
+        if len(paths) != 2 or not all(_mentions_bios(v) for v in paths[0]):
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}", f"link without a BIOS source: {event.raw}"
+            )
+        return True
+
+    @staticmethod
+    def _same_link_run(previous: Event, event: Event) -> bool:
+        return (
+            event.words[0][0] == "ln"
+            and previous.path == event.path
+            and event.line - previous.line <= 2
+        )
+
+    def _links(self, events: list[Event]) -> None:
+        sources: list[str] = []
+        targets: list[str] = []
+        for event in events:
+            paths = [w for w in event.words[1:] if not w[0].startswith("-")]
+            source, target = paths
+            if len(target) == 1 and len(source) > 1:
+                target = [f"{target[0].rstrip('/')}/{_split_path(s)[1]}" for s in source]
+            sources += source
+            targets += target
+        folders = {_split_path(s)[0] for s in sources}
+        whole_dir = all(
+            not _is_file(s) and "/" not in s.replace(BIOSDIR, "", 1).strip("/") for s in sources
+        )
+        if whole_dir:
+            names: list[str] = []
+            search_dir = sources[0]
+            passes_to = f"{targets[0]} is a symlink to it"
+        else:
+            if len(folders) != 1:
+                raise LauncherPatternMissing(
+                    f"{events[0].path}:{events[0].line}",
+                    "links from several BIOS folders in one run",
+                )
+            search_dir = folders.pop()
+            names = [_split_path(s)[1] for s in sources]
+            passes_to = _render_targets(names, targets)
+        self.record(
+            "links", names, [(e.path, e.line) for e in events],
+            case_insensitive=False, search_dir=search_dir, passes_to=passes_to,
+            fails_when_missing=False, depends_on_option=self._depends(events[0]),
+        )
+
+    def _copy(self, event: Event) -> None:
+        paths = [v for w in event.words[1:] if not w[0].startswith("-") for v in w]
+        *sources, target = paths
+        if not _mentions_bios(target):
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}", f"copy out of the BIOS directory: {event.raw}"
+            )
+        folders = {_split_path(s)[0] for s in sources}
+        if len(folders) != 1:
+            raise LauncherPatternMissing(f"{event.path}:{event.line}", "copy from several folders")
+        self.record(
+            "copies", [_split_path(s)[1] for s in sources], [(event.path, event.line)],
+            search_dir=folders.pop(), passes_to=target.rstrip("/"),
+            fails_when_missing=False, depends_on_option=self._depends(event),
+        )
+
+    def _run_cmd(self, event: Event) -> None:
+        inner = Event(event.path, event.line, event.words[1:], event.raw, event.conditions, False)
+        if not inner.words or inner.words[0][0] != "unzip":
+            raise LauncherPatternMissing(f"{event.path}:{event.line}", f"runCmd of {event.raw}")
+        self._unzip(inner)
+
+    def _unzip(self, event: Event) -> None:
+        values = [v for w in event.words[1:] for v in w]
+        if "-d" not in values:
+            raise LauncherPatternMissing(f"{event.path}:{event.line}", "unzip without -d")
+        target = values[values.index("-d") + 1]
+        archives = [v for v in values if v.endswith(".zip")]
+        if len(archives) != 1 or not _mentions_bios(target):
+            raise LauncherPatternMissing(f"{event.path}:{event.line}", f"unzip of {event.raw}")
+        self.record(
+            "copies", [], [(event.path, event.line)],
+            search_dir=archives[0], passes_to=target,
+            fails_when_missing=False, depends_on_option=self._depends(event),
+        )
+
+    def _move(self, event: Event) -> None:
+        source, target = (w for w in event.words[1:] if not w[0].startswith("-"))
+        if not all(_mentions_bios(v) for v in source + target):
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}", f"move across folders: {event.raw}"
+            )
+        folders = {_split_path(s)[0] for s in source}
+        destinations = {_split_path(t)[0] for t in target}
+        if len(folders) != 1 or len(destinations) != 1:
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}", "move between several folders"
+            )
+        self.record(
+            "moves", [_split_path(s)[1] for s in source], [(event.path, event.line)],
+            case_insensitive=False, search_dir=folders.pop(), passes_to=destinations.pop(),
+            fails_when_missing=False, depends_on_option=self._depends(event),
+        )
+
+    def _move_dir(self, event: Event) -> None:
+        link, real = (w[0] for w in event.words[1:3])
+        if not _mentions_bios(real) or _mentions_bios(link):
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}", f"moveConfigDir into a non-BIOS folder: {event.raw}"
+            )
+        self.record(
+            "links", [], [(event.path, event.line)],
+            search_dir=real, passes_to=f"{link} is a symlink to it",
+            fails_when_missing=False, depends_on_option=self._depends(event),
+        )
+
+    def _ini_set(self, event: Event) -> None:
+        key, value = event.words[1][0], event.words[2][0]
+        target = self._ini_file(event)
+        passes_to = f"{key} in {self.final(target)}" if target else key
+        if _is_file(value) and ";" not in value:
+            folder, name = _split_path(value)
+            self.record(
+                "fixed_name", [name], [(event.path, event.line)],
+                case_insensitive=False, search_dir=folder, passes_to=passes_to,
+                fails_when_missing=False, depends_on_option=self._depends(event),
+            )
+            return
+        self.record(
+            "passes_path", [], [(event.path, event.line)],
+            search_dir=value, passes_to=passes_to,
+            fails_when_missing=False, depends_on_option=self._depends(event),
+        )
+
+    def _ini_file(self, event: Event) -> str | None:
+        """The file the closest iniConfig above this iniSet points at."""
+        if len(event.words) > 3:
+            return event.words[3][0]
+        lines = self.scripts.files[event.path].splitlines()
+        for number in range(event.line - 1, 0, -1):
+            words = _split(lines[number - 1].strip())
+            if words and words[0] == "iniConfig" and len(words) >= 4:
+                return words[3]
+            if _FUNCTION.match(lines[number - 1]):
+                break
+        return None
+
+    def _written(self, event: Event) -> None:
+        """A file the package writes into the BIOS directory itself."""
+        target = next(
+            (v for w in reversed(event.words[1:]) for v in w if _mentions_bios(v)), None
+        )
+        if target is None or not _is_file(target):
+            raise LauncherPatternMissing(f"{event.path}:{event.line}", f"write of {event.raw}")
+        folder, name = _split_path(target)
+        self.record(
+            "copies", [name], [(event.path, event.line)],
+            search_dir=None, passes_to=folder,
+            fails_when_missing=False, depends_on_option=self._depends(event),
+        )
+
+    def _build_flag(self, event: Event) -> None:
+        flags = [
+            v for w in event.words[1:] for v in w
+            if v.startswith("-D") and "=" in v and _mentions_bios(v)
+        ]
+        if not flags:
+            raise LauncherPatternMissing(f"{event.path}:{event.line}", f"cmake of {event.raw}")
+        for flag in flags:
+            option, value = flag.split("=", 1)
+            self.record(
+                "passes_path", [], [(event.path, event.line)],
+                search_dir=value, passes_to=f"build option {option}",
+                fails_when_missing=False, depends_on_option=self._depends(event),
+            )
+
+    def _heredoc_setting(self, event: Event) -> None:
+        """A `KEY = $biosdir/...` or `--flag="$biosdir"` line of a generated file."""
+        text = event.raw
+        setting = re.match(r'^([\w.-]+)\s*=\s*"?([^"]*)"?$', text)
+        flag = re.match(r'^(--?[\w-]+)(?:=|\s+)"?([^"]*)"?$', text)
+        found = flag or setting
+        if found is None:
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}", f"generated line names the BIOS directory: {text}"
+            )
+        key, value = found.group(1), found.group(2)
+        kind, names, folder = "passes_path", [], value
+        if _is_file(value):
+            folder, name = _split_path(value)
+            kind, names = "fixed_name", [name]
+        written = _redirect_target(event.opener)
+        if written is None:
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}", "generated file with no > or >> target"
+            )
+        self.record(
+            kind, names, [(event.path, event.line)],
+            case_insensitive=False if kind == "fixed_name" else None,
+            search_dir=folder, passes_to=f"{key} in {self.final(written)}",
+            fails_when_missing=False, depends_on_option=self._depends(event),
+        )
+
+    def _add_emulator(self, event: Event) -> None:
+        if len(event.words) < 5:
+            return
+        command = event.words[4][0]
+        if not _mentions_bios(command):
+            return
+        names = event.words[2]
+        tokens = _split(command)
+        found = [
+            (flag, value) for flag, value in zip(tokens, tokens[1:])
+            if flag.startswith("-") and _mentions_bios(value)
+        ]
+        found += [
+            tuple(token.split("=", 1)) for token in tokens
+            if token.startswith("-") and "=" in token and _mentions_bios(token.split("=", 1)[1])
+        ]
+        if not found:
+            raise LauncherPatternMissing(
+                f"{event.path}:{event.line}",
+                f"addEmulator names the BIOS directory without a flag: {command}",
+            )
+        for emulator in names:
+            for flag, value in found:
+                self._command_line(event, emulator, flag, value)
+
+    def _command_line(self, event: Event, emulator: str, flag: str, value: str) -> None:
+        folder, name = _split_path(value)
+        if _is_file(value):
+            kind, names, search = "fixed_name", [name], folder
+        else:
+            kind, names, search = "passes_path", [], value
+        self.record(
+            kind, names, [(event.path, event.line)], emulator=emulator,
+            case_insensitive=False if kind == "fixed_name" else None,
+            search_dir=search, passes_to=f"command line {flag}",
+            fails_when_missing=False, depends_on_option=self._depends(event),
+        )
+
+    def _outside_bios(self, events: list[Event]) -> None:
+        flags = OUTSIDE_BIOS.get(self.package.id)
+        if not flags:
+            return
+        seen: set[str] = set()
+        for event in events:
+            if event.words[0][0] != "addEmulator" or len(event.words) < 5:
+                continue
+            tokens = _split(event.words[4][0])
+            for flag, value in zip(tokens, tokens[1:]):
+                if flag in flags:
+                    seen.add(flag)
+                    for emulator in event.words[2]:
+                        self._command_line(event, emulator, flag, value)
+        missing = sorted(set(flags) - seen)
+        if missing:
+            raise LauncherPatternMissing(
+                self.package.path, f"addEmulator no longer passes {missing}"
+            )
+
+
+def _redirect_target(words: list[list[str]]) -> str | None:
+    """The file a `cat > FILE << EOF` or `cat << EOF >> FILE` line writes."""
+    flat = [w[0] for w in words if w]
+    for index, word in enumerate(flat[:-1]):
+        if word in (">", ">>"):
+            return flat[index + 1]
+    return None
+
+
+def merge_records(records: list[Launcher]) -> list[Launcher]:
+    """Records equal but for their lines become one, citing every line."""
+    merged: dict[tuple[Any, ...], Launcher] = {}
+    for record in records:
+        entry = record.entry()
+        entry.pop("source_ref")
+        key = tuple((k, repr(v)) for k, v in entry.items())
+        if key in merged:
+            kept = merged[key]
+            kept.lines += record.lines
+            kept.source_ref = source_ref(kept.lines)
+        else:
+            merged[key] = record
+    return list(merged.values())
+
+
+def _render_targets(names: list[str], targets: list[str]) -> str:
+    """Where linked names land: one folder when the names are kept, each path when renamed."""
+    folders = {_split_path(t)[0] for t in targets}
+    kept = [_split_path(t)[1] for t in targets] == names
+    if kept and len(folders) == 1:
+        return f"{folders.pop()}/<name>"
+    return ", ".join(targets)
+
+
+def source_ref(lines: list[tuple[str, int]]) -> str:
+    """`path:a-b, c` per file, in first-seen file order, adjacent lines joined."""
+    by_path: dict[str, list[int]] = {}
+    for path, number in lines:
+        by_path.setdefault(path, []).append(number)
+    parts = []
+    for path, numbers in by_path.items():
+        ranges: list[list[int]] = []
+        for number in sorted(set(numbers)):
+            if ranges and number == ranges[-1][1] + 1:
+                ranges[-1][1] = number
+            else:
+                ranges.append([number, number])
+        spans = ", ".join(str(a) if a == b else f"{a}-{b}" for a, b in ranges)
+        parts.append(f"{path}:{spans}")
+    return ", ".join(parts)
+
+
+def package_record(scripts: Scripts, package: Module) -> Launcher:
+    """What the package builds and the systems it registers."""
+    systems: list[str] = []
+    lines: list[tuple[str, int]] = [(package.path, package.id_line)]
+    if package.repo_line is not None:
+        lines.append((package.path, package.repo_line))
+    for event in scripts.walk(package):
+        if event.heredoc or not event.words or not event.words[0]:
+            continue
+        command = event.words[0][0]
+        if command == "addSystem":
+            values = event.words[1] if len(event.words) > 1 else []
+        elif command == "addPort":
+            values = ["ports"]
+        else:
+            continue
+        if not values:
+            raise LauncherPatternMissing(f"{event.path}:{event.line}", f"{command} takes no value")
+        for value in values:
+            if "$" in value or not value:
+                raise LauncherPatternMissing(
+                    f"{event.path}:{event.line}", f"system name {value!r} not resolved"
+                )
+            if value not in systems:
+                systems.append(value)
+        lines.append((event.path, event.line))
+    return Launcher(
+        emulator=package.id, package=package.id, repo=package.repo,
+        systems=systems or None, build=package.build, kind="package", names=[],
+        source_ref=source_ref(lines),
+    )
+
+
+def launch_script_records(scripts: Scripts) -> list[Launcher]:
+    """The launch scripts installed next to an emulator that refuse to start without a file."""
+    records = []
+    for path, text in sorted(scripts.files.items()):
+        parts = PurePosixPath(path).parts
+        is_script = (
+            len(parts) == 4 and parts[0] == SCRIPTMODULES and parts[1] in SECTIONS
+            and path.endswith(".sh")
+        )
+        if not is_script or not _mentions_bios(text):
+            continue
+        package = scripts.modules.get(parts[2])
+        if package is None:
+            raise LauncherPatternMissing(path, f"launch script of no package {parts[2]}")
+        records += _launch_script(path, text.splitlines(), package)
+    return records
+
+
+def _launch_script(path: str, lines: list[str], package: Module) -> list[Launcher]:
+    env: dict[str, str] = {"HOME": "$HOME"}
+    records = []
+    checks = 0
+    number = 0
+    while number < len(lines):
+        number += 1
+        stripped = lines[number - 1].strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        assign = re.match(r'^([A-Za-z_]\w*)="([^"]*)"$', stripped)
+        if assign is not None:
+            env[assign.group(1)] = _VAR.sub(
+                lambda m: env.get(m.group(1) or m.group(3), m.group(0)), assign.group(2)
+            )
+            continue
+        check = re.match(r'^if \[\[ ! -f "([^"]+)" \]\]; then$', stripped)
+        if check is None:
+            continue
+        tested = check.group(1)
+        resolved = _VAR.sub(lambda m: env.get(m.group(1) or m.group(3), m.group(0)), tested)
+        if not _mentions_bios(resolved):
+            continue
+        block_start = number
+        message: str | None = None
+        exits = False
+        while number < len(lines) and lines[number].strip() != "fi":
+            number += 1
+            body = lines[number - 1].strip()
+            exits = exits or body == "exit 1"
+            pause = re.search(r'--pause "([^"]*)"', body)
+            if pause is not None:
+                message = pause.group(1)
+        number += 1
+        if not exits:
+            continue
+        checks += 1
+        variable = re.fullmatch(r"\$\{?(\w+)\}?", tested)
+        passes_to = None
+        cited = list(range(block_start, number + 1))
+        if variable is not None:
+            for later, line in enumerate(lines[number:], number + 1):
+                flag = re.search(r'(--?[\w-]+)="\$' + variable.group(1) + r'"', line)
+                if flag is not None:
+                    passes_to = f"command line {flag.group(1)}"
+                    cited.append(later)
+                    break
+        folder, name = _split_path(resolved)
+        records.append(Launcher(
+            emulator=package.id, package=package.id, build=package.build,
+            kind="fixed_name", names=[name], case_insensitive=False,
+            search_dir=folder, passes_to=passes_to,
+            fails_when_missing=message if message is not None else True,
+            source_ref=source_ref([(path, n) for n in cited]),
+        ))
+    if not checks:
+        raise LauncherPatternMissing(path, "names the BIOS directory but checks no file")
+    return records
+
+
+def extract_launchers(files: dict[str, str]) -> list[dict[str, Any]]:
+    """Every launcher record, from the scriptmodules keyed by repo-relative path."""
+    scripts = Scripts(files)
+    records: list[Launcher] = []
+    for package in sorted(scripts.modules.values(), key=lambda m: m.path):
+        records.append(package_record(scripts, package))
+        records += merge_records(PackageReader(scripts, package).read())
+    records += launch_script_records(scripts)
+    return [record.entry() for record in records]

scripts/scraper/retropie_scraper.py

@@ -15,6 +15,10 @@ RetroArch: RetroPie claimed every libretro core in existence while shipping
 96 of them, and claimed none of the standalone emulators it also packages
 (openmsx, pcsx2, xroar, sdltrs, amiberry and the rest). Both halves were
 wrong, in opposite directions.
+
+The same scripts carry the launcher layer (retropie_launchers): where each
+package links, copies or passes a BIOS file, what it builds and which
+systems it registers.
 """
 
 from __future__ import annotations
@@ -27,8 +31,14 @@ import urllib.request
 
 try:
     from .base_scraper import BaseScraper, BiosRequirement, fetch_branch_head
+    from .retropie_launchers import SCRIPTMODULES
+    from .retropie_launchers import SECTIONS as _SCRIPT_SECTIONS
+    from .retropie_launchers import extract_launchers
 except ImportError:
     from base_scraper import BaseScraper, BiosRequirement, fetch_branch_head
+    from retropie_launchers import SCRIPTMODULES
+    from retropie_launchers import SECTIONS as _SCRIPT_SECTIONS
+    from retropie_launchers import extract_launchers
 
 PLATFORM_NAME = "retropie"
 
@@ -43,7 +53,7 @@ _VERSION_FILE = "retropie_packages.sh"
 _VERSION = re.compile(r'^__version="([^"]+)"', re.MULTILINE)
 # Sections RetroPie does not build as emulators: setup helpers, themes,
 # drivers and the like carry no core.
-_PACKAGE_DIRS = ("emulators", "libretrocores", "ports")
+_PACKAGE_DIRS = _SCRIPT_SECTIONS
 
 
 class Scraper(BaseScraper):
@@ -56,6 +66,8 @@ class Scraper(BaseScraper):
         super().__init__(url=url)
         self._modules: list[str] | None = None
         self._version = ""
+        # Package scripts and the launch scripts beside them, by repo path.
+        self._scripts: dict[str, str] = {}
 
     def _fetch_archive(self) -> bytes:
         request = urllib.request.Request(
@@ -89,12 +101,19 @@ class Scraper(BaseScraper):
                     and parts[0] == "scriptmodules"
                     and parts[1] in _PACKAGE_DIRS
                 )
-                if not (is_version_file or is_package):
+                is_launch_script = (
+                    len(parts) == 4 and parts[0] == SCRIPTMODULES and parts[1] in _SCRIPT_SECTIONS
+                )
+                if not (is_version_file or is_package or is_launch_script):
                     continue
                 handle = archive.extractfile(member)
                 if handle is None:
                     continue
                 text = handle.read().decode("utf-8", errors="replace")
+                if is_package or is_launch_script:
+                    self._scripts[relative] = text
+                if is_launch_script:
+                    continue
                 match = (_VERSION if is_version_file else _MODULE_ID).search(text)
                 if not match:
                     continue
@@ -139,6 +158,7 @@ class Scraper(BaseScraper):
             "source": self.url,
             "base_destination": "BIOS",
             "cores": cores,
+            "launchers": extract_launchers(self._scripts),
         }