feat: read ROCKNIX launchers from its packages

The scraper reads the start scripts and the configs each package installs at the tag the BIOS list is pinned to: configs that name a BIOS path, the yabasanshiro search, directories handed to DuckStation and the PCSX2 builds, and the ln -s commands that put bios/ where an emulator reads. A package that ships one config per device gets a device field only where the values differ, as melonDS does on AMD64.

scripts/scraper/batocera_launchers.py

@@ -50,6 +50,7 @@ class Launcher:
     fails_when_missing: bool | str = False
     fails_scope: str | None = None
     depends_on_option: str | None = None
+    device: str | None = None
 
     def entry(self) -> dict[str, Any]:
         """The platform YAML form, in a fixed key order, unset keys left out."""
@@ -57,7 +58,7 @@ class Launcher:
             "emulator", "build", "kind", "slot", "names", "expands",
             "case_insensitive", "search_dir", "passes_to",
             "fails_when_missing", "fails_scope", "depends_on_option",
-            "source_ref",
+            "device", "source_ref",
         )
         return {key: getattr(self, key) for key in order if getattr(self, key) is not None}
 

scripts/scraper/rocknix_launchers.py

@@ -0,0 +1,648 @@
+"""ROCKNIX's launcher layer, read from its packages at the pinned tag.
+
+ROCKNIX starts each standalone emulator from a shell script and installs a
+config with the package. Between a file in /storage/roms/bios and the
+emulator, the config names the path (melonDS, xemu, mednafen), the script
+searches or links (yabasanshiro, drastic, dolphin), or the emulator is
+handed a directory and left to choose (DuckStation, AetherSX2). Several
+packages ship one config per device: a record carries `device` only where
+the value read differs between them.
+"""
+
+from __future__ import annotations
+
+import posixpath
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+
+try:
+    from .batocera_launchers import Launcher
+    from .shell_launchers import (
+        LauncherPatternMissing,
+        Link,
+        RepoTree,
+        Script,
+        Setting,
+        describe_keys,
+        settings,
+        source_ref,
+    )
+except ImportError:
+    from batocera_launchers import Launcher
+    from shell_launchers import (
+        LauncherPatternMissing,
+        Link,
+        RepoTree,
+        Script,
+        Setting,
+        describe_keys,
+        settings,
+        source_ref,
+    )
+
+ROCKNIX = "projects/ROCKNIX"
+DEVICES = f"{ROCKNIX}/devices"
+STANDALONE = f"{ROCKNIX}/packages/emulators/standalone"
+RETROARCH = f"{ROCKNIX}/packages/emulators/libretro/retroarch"
+BIOS_ROOT = "/storage/roms/bios"
+
+Lines = list[tuple[str, set[int]]]
+
+
+@dataclass
+class DeviceConfig:
+    """The config file one or more devices install, and the lines that pick it."""
+
+    path: str
+    devices: list[str]
+    mk_lines: set[int]
+
+
+def device_configs(
+    tree: RepoTree, package: str, file: str, folder: str = "config"
+) -> list[DeviceConfig]:
+    """Each distinct file package.mk installs as `file` for some device.
+
+    The directory comes from `${PKG_DIR}/<folder>/${DEVICE}/` or from a
+    `case ${DEVICE}` naming it. A device whose directory is absent does not
+    get the package. A directory that is a symlink resolves to its target.
+    """
+    mk_path = f"{package}/package.mk"
+    mk = Script(mk_path, tree.read(mk_path))
+    arms: list[tuple[list[str], str]] = []
+    mk_lines: set[int] = set()
+    for block in re.finditer(r'case\s+"?\$\{?DEVICE\}?"?\s+in(.*?)\besac\b', mk.text, re.S):
+        body = block.group(1)
+        if not re.search(rf"CONFIG=|\$\{{PKG_DIR\}}/{folder}/", body):
+            continue
+        offset = block.start(1)
+        for arm in re.finditer(r"^\s*([\w*|]+)\)(.*?);;", body, re.M | re.S):
+            chosen = re.search(
+                rf'CONFIG="([^"]+)"|\$\{{PKG_DIR\}}/{folder}/([^/\s]+)', arm.group(2)
+            )
+            if chosen:
+                arms.append((arm.group(1).split("|"), chosen.group(1) or chosen.group(2)))
+                mk_lines.add(mk.text.count("\n", 0, offset + arm.start(2)) + 1)
+        break
+    if not arms:
+        number, _ = mk.find(
+            rf"\$\{{PKG_DIR\}}/{folder}/\$\{{DEVICE\}}/", f"{folder}/${{DEVICE}} copy"
+        )
+        arms, mk_lines = [(["*"], "${DEVICE}")], {number}
+    root = f"{package}/{folder}"
+    entries = tree.listdir(root)
+    by_path: dict[str, DeviceConfig] = {}
+    for device, entry in tree.listdir(DEVICES).items():
+        if entry.kind != "dir":
+            continue
+        arm = next((d for patterns, d in arms if device in patterns or "*" in patterns), None)
+        if arm is None:
+            continue
+        name = device if arm in ("${DEVICE}", "${CONFIG}") else arm
+        found = entries.get(name)
+        if found is None:
+            continue
+        if found.kind == "link":
+            name = found.target.rstrip("/")
+        path = f"{root}/{name}/{file}"
+        by_path.setdefault(path, DeviceConfig(path, [], mk_lines)).devices.append(device)
+    if not by_path:
+        raise LauncherPatternMissing(root, f"{file} for any device")
+    return sorted(by_path.values(), key=lambda c: c.path)
+
+
+def per_device(
+    tree: RepoTree,
+    configs: list[DeviceConfig],
+    mk_path: str,
+    read: Callable[[str, str], tuple[object, set[int]]],
+) -> list[tuple[str | None, object, Lines]]:
+    """Configs grouped by what read() finds in them.
+
+    read(path, text) returns a comparable value and the lines it comes from.
+    One group gives one result without a device. Otherwise each group names
+    its devices, in the order of the devices directory.
+    """
+    groups: dict[object, tuple[list[str], Lines]] = {}
+    for config in configs:
+        value, lines = read(config.path, tree.read(config.path))
+        devices, refs = groups.setdefault(value, ([], []))
+        devices.extend(config.devices)
+        refs.append((config.path, lines))
+    mk_lines = set().union(*(c.mk_lines for c in configs))
+    if len(groups) == 1:
+        ((value, (_, refs)),) = groups.items()
+        return [(None, value, [*refs, (mk_path, mk_lines)])]
+    order = list(tree.listdir(DEVICES))
+    return [
+        (", ".join(sorted(devices, key=order.index)), value, [*refs, (mk_path, mk_lines)])
+        for value, (devices, refs) in sorted(
+            groups.items(), key=lambda g: min(order.index(d) for d in g[1][0])
+        )
+    ]
+
+
+def _file_under(value: str, root: str) -> bool:
+    return value.startswith(root.rstrip("/") + "/")
+
+
+@dataclass(frozen=True)
+class FixedPaths:
+    """Keys of a config that name a file under one directory."""
+
+    keys: tuple[tuple[str, str], ...]
+    names: tuple[str, ...]
+    search_dir: str
+
+
+def fixed_paths(
+    found: list[Setting], keys: list[tuple[str, str]], path: str
+) -> tuple[FixedPaths, set[int]]:
+    """The values of keys, as names relative to the directory they share."""
+    by_key = {(s.section, s.key): s for s in found}
+    missing = [k for k in keys if k not in by_key]
+    if missing:
+        raise LauncherPatternMissing(path, f"key {missing[0][1]}")
+    values = [by_key[k].value for k in keys if by_key[k].value]
+    lines = {by_key[k].line for k in keys}
+    if not values:
+        return FixedPaths(tuple(keys), (), ""), lines
+    folder = posixpath.commonpath([posixpath.dirname(v) for v in values])
+    names = tuple(posixpath.relpath(v, folder) for v in values)
+    return FixedPaths(tuple(keys), names, folder), lines
+
+
+def bios_keys(tree: RepoTree, paths: list[str], root: str = BIOS_ROOT) -> list[tuple[str, str]]:
+    """Keys whose value is a file under root in at least one of the configs."""
+    keys: list[tuple[str, str]] = []
+    for path in paths:
+        for setting in settings(tree.read(path)):
+            key = (setting.section, setting.key)
+            if _file_under(setting.value, root) and key not in keys:
+                keys.append(key)
+    if not keys:
+        raise LauncherPatternMissing(paths[0], f"key naming a file under {root}")
+    return keys
+
+
+def config_records(
+    tree: RepoTree,
+    emulator: str,
+    package: str,
+    file: str,
+    passes_to_suffix: str = "",
+    **fields: object,
+) -> list[Launcher]:
+    """fixed_name records for the bios paths of a config installed per device."""
+    configs = device_configs(tree, package, file)
+    keys = bios_keys(tree, [c.path for c in configs])
+    name = posixpath.basename(file)
+
+    def read(path: str, text: str) -> tuple[object, set[int]]:
+        return fixed_paths(settings(text), keys, path)
+
+    records = []
+    for device, value, refs in per_device(tree, configs, f"{package}/package.mk", read):
+        assert isinstance(value, FixedPaths)
+        described = describe_keys(name, value.keys)
+        records.append(
+            Launcher(
+                emulator=emulator,
+                build="standalone",
+                kind="fixed_name",
+                names=list(value.names),
+                source_ref=source_ref(refs),
+                search_dir=value.search_dir or None,
+                passes_to=(
+                    (described + passes_to_suffix) if value.names else f"{described} left empty"
+                ),
+                device=device,
+                **fields,
+            )
+        )
+    return records
+
+
+def retroarch(tree: RepoTree) -> list[Launcher]:
+    """system_directory of the retroarch.cfg each device installs."""
+    configs = device_configs(tree, RETROARCH, "retroarch.cfg", folder="sources")
+    wanted = ("system_directory", "systemfiles_in_content_dir")
+
+    def read(path: str, text: str) -> tuple[object, set[int]]:
+        found = {s.key: s for s in settings(text) if s.key in wanted}
+        if "system_directory" not in found:
+            raise LauncherPatternMissing(path, "system_directory")
+        return tuple((k, found[k].value) for k in wanted if k in found), {
+            s.line for s in found.values()
+        }
+
+    records = []
+    for device, value, refs in per_device(tree, configs, f"{RETROARCH}/package.mk", read):
+        assert isinstance(value, tuple)
+        values = dict(value)
+        records.append(
+            Launcher(
+                emulator="retroarch",
+                build="libretro",
+                kind="passes_path",
+                names=[],
+                source_ref=source_ref(refs),
+                search_dir=values["system_directory"],
+                passes_to="retroarch.cfg " + ", ".join(f'{k} = "{v}"' for k, v in value),
+                device=device,
+            )
+        )
+    return records
+
+
+def amiberry(tree: RepoTree) -> list[Launcher]:
+    """The A500 or A1200 config, chosen by the game name, then the kickstarts link."""
+    package = f"{STANDALONE}/amiberry"
+    script = Script(
+        f"{package}/scripts/start_amiberry.sh", tree.read(f"{package}/scripts/start_amiberry.sh")
+    )
+    test_line, test = script.find(r"grep -E '([^']+)'", "machine choice by game name")
+    then_line, then = script.find(
+        r"cp \S+/(\w+-default\.uae)", "config copied for a match", test_line
+    )
+    else_line, _ = script.find(r"^\s*else\b", "else of the machine choice", then_line)
+    other_line, other = script.find(
+        r"cp \S+/(\w+-default\.uae)", "config copied otherwise", else_line
+    )
+    sed_line, sed = script.find(
+        r'sed -i "s#(@\w+@)#\$\{(\w+)\}#g"', "placeholder substitution", other_line
+    )
+    used: set[int] = {test_line, then_line, else_line, other_line, sed_line}
+    root = script.expand("${" + sed.group(2) + "}", sed_line, used)
+    records = []
+    for config, condition in (
+        (then.group(1), f"game name matches {test.group(1)}"),
+        (other.group(1), f"game name does not match {test.group(1)}"),
+    ):
+        path = f"{package}/config/conf/{config}"
+        found = [
+            Setting(s.section, s.key, s.value.replace(sed.group(1), root), s.line)
+            for s in settings(tree.read(path))
+        ]
+        keys = [(s.section, s.key) for s in found if _file_under(s.value, root)]
+        if not keys:
+            raise LauncherPatternMissing(path, f"key naming a file under {sed.group(1)}")
+        value, lines = fixed_paths(found, keys, path)
+        slot = re.fullmatch(r"Amiga(\w+)-default\.uae", config)
+        records.append(
+            Launcher(
+                emulator="amiberry",
+                build="standalone",
+                kind="fixed_name",
+                names=list(value.names),
+                source_ref=source_ref([(script.path, used), (path, lines)]),
+                slot=slot.group(1) if slot else config,
+                search_dir=value.search_dir,
+                passes_to=f"{describe_keys(config, value.keys)} (amiberry -f)",
+                depends_on_option=condition,
+            )
+        )
+    mk = Script(f"{package}/package.mk", tree.read(f"{package}/package.mk"))
+    links = mk.links(BIOS_ROOT, fixed={"INSTALL": ""})
+    if not links:
+        raise mk.missing(f"ln -s of {BIOS_ROOT}")
+    records.extend(link_record("amiberry", mk, link) for link in links)
+    return records
+
+
+def link_record(emulator: str, script: Script, link: Link, **fields: object) -> Launcher:
+    return Launcher(
+        emulator=emulator,
+        build="standalone",
+        kind="links",
+        names=list(link.names),
+        source_ref=source_ref([(script.path, link.lines)]),
+        expands=link.expands or None,
+        search_dir=link.bios_side,
+        passes_to=f"symlink {link.link} -> {link.target}",
+        **fields,
+    )
+
+
+def hatari(tree: RepoTree) -> list[Launcher]:
+    package = f"{STANDALONE}/hatarisa"
+    script = Script(
+        f"{package}/scripts/start_hatari.sh", tree.read(f"{package}/scripts/start_hatari.sh")
+    )
+    copy_line, copy = script.find(r"cp \S+/([\w-]+\.cfg) ", "default config copy")
+    run_line, _ = script.find(r"--configfile", "--configfile", copy_line)
+    path = f"{package}/config/conf/{copy.group(1)}"
+    found = settings(tree.read(path))
+    keys = [(s.section, s.key) for s in found if _file_under(s.value, BIOS_ROOT)]
+    if not keys:
+        raise LauncherPatternMissing(path, f"key naming a file under {BIOS_ROOT}")
+    value, lines = fixed_paths(found, keys, path)
+    return [
+        Launcher(
+            emulator="hatari",
+            build="standalone",
+            kind="fixed_name",
+            names=list(value.names),
+            source_ref=source_ref([(path, lines), (script.path, {copy_line, run_line})]),
+            search_dir=value.search_dir,
+            passes_to=f"{describe_keys(copy.group(1), value.keys)} (--configfile)",
+        )
+    ]
+
+
+def melonds(tree: RepoTree) -> list[Launcher]:
+    package = f"{STANDALONE}/melonds-sa"
+    script = Script(
+        f"{package}/scripts/start_melonds.sh", tree.read(f"{package}/scripts/start_melonds.sh")
+    )
+    enable_line, _ = script.find(r"ExternalBIOSEnable=1", "ExternalBIOSEnable=1")
+    tests = [
+        (number, found)
+        for number, line in script.code()
+        if number < enable_line and (found := re.search(r'if \[ "\$(\w+)" = "(\w+)" \]', line))
+    ]
+    if not tests:
+        raise script.missing("test that enables the external BIOS")
+    test_line, test = tests[-1]
+    setting_line, setting = script.find(
+        rf"^\s*{test.group(1)}=\$\(get_setting (\w+) ", f"setting read into {test.group(1)}"
+    )
+    condition = f"{setting.group(1)} = {test.group(2)} sets ExternalBIOSEnable=1, otherwise 0"
+    records = config_records(tree, "melonds", package, "melonDS.ini", depends_on_option=condition)
+    for record in records:
+        record.source_ref += (
+            f", {source_ref([(script.path, {setting_line, test_line, enable_line})])}"
+        )
+    return records
+
+
+def dsperate(tree: RepoTree) -> list[Launcher]:
+    package = f"{STANDALONE}/dsperate-sa"
+    script = Script(
+        f"{package}/scripts/start_dsperate.sh", tree.read(f"{package}/scripts/start_dsperate.sh")
+    )
+    run_line, _ = script.find(r"/usr/bin/dsperate --config ", "dsperate --config")
+    records = config_records(tree, "dsperate", package, "dsperate.ini", " (--config)")
+    for record in records:
+        record.source_ref += f", {source_ref([(script.path, {run_line})])}"
+    return records
+
+
+def xemu(tree: RepoTree) -> list[Launcher]:
+    package = f"{STANDALONE}/xemu-sa"
+    script = Script(
+        f"{package}/scripts/start_xemu.sh", tree.read(f"{package}/scripts/start_xemu.sh")
+    )
+    copy_line, _ = script.find(
+        r'cp -r "\$\{SOURCE_DIR\}/\$\{XEMU_INI\}"', "xemu.toml copy at launch"
+    )
+    records = config_records(tree, "xemu", package, "xemu.toml", " (copied at each launch)")
+    for record in records:
+        record.source_ref += f", {source_ref([(script.path, {copy_line})])}"
+    return records
+
+
+def nanoboyadvance(tree: RepoTree) -> list[Launcher]:
+    package = f"{STANDALONE}/nanoboyadvance-sa"
+    path = f"{package}/config/config.toml"
+    found = settings(tree.read(path))
+    keys = [(s.section, s.key) for s in found if _file_under(s.value, BIOS_ROOT)]
+    if not keys:
+        raise LauncherPatternMissing(path, f"key naming a file under {BIOS_ROOT}")
+    value, lines = fixed_paths(found, keys, path)
+    mk_path = f"{package}/package.mk"
+    mk_line, _ = Script(mk_path, tree.read(mk_path)).find(
+        r"config/config\.toml", "config.toml install"
+    )
+    return [
+        Launcher(
+            emulator="nanoboyadvance",
+            build="standalone",
+            kind="fixed_name",
+            names=list(value.names),
+            source_ref=source_ref([(path, lines), (mk_path, {mk_line})]),
+            search_dir=value.search_dir,
+            passes_to=describe_keys("config.toml", value.keys),
+        )
+    ]
+
+
+def mednafen(tree: RepoTree) -> list[Launcher]:
+    """The firmware keys of mednafen.template, one record per module."""
+    package = f"{STANDALONE}/mednafen"
+    path = f"{package}/config/common/mednafen.template"
+    found = settings(tree.read(path), separator=" ")
+    folder = next((s for s in found if s.key == "filesys.path_firmware"), None)
+    if folder is None:
+        raise LauncherPatternMissing(path, "filesys.path_firmware")
+    pattern = re.compile(r"(\w+)\.(\w*bios\w*|ggrom|cart\.\w+_path)")
+    firmware = [s for s in found if pattern.fullmatch(s.key) and s.value and not s.value.isdigit()]
+    if not firmware:
+        raise LauncherPatternMissing(path, "firmware keys")
+    script_path = f"{package}/scripts/start_mednafen.sh"
+    script = Script(script_path, tree.read(script_path))
+    gen_line, _ = script.find(r"mednafen_gen_config\.sh", "config generation")
+    test_line, _ = script.find(
+        r'if \[ ! -f "\$MEDNAFEN_HOME/mednafen\.cfg" \]', "mednafen.cfg test"
+    )
+    modules: dict[str, list[Setting]] = {}
+    for setting in firmware:
+        modules.setdefault(setting.key.split(".", 1)[0], []).append(setting)
+    return [
+        Launcher(
+            emulator="mednafen",
+            build="standalone",
+            kind="fixed_name",
+            names=list(dict.fromkeys(s.value for s in group)),
+            source_ref=source_ref(
+                [
+                    (path, {folder.line, *(s.line for s in group)}),
+                    (script_path, {test_line, gen_line}),
+                ]
+            ),
+            slot=module,
+            search_dir=folder.value,
+            passes_to="mednafen.cfg " + ", ".join(s.key for s in group),
+        )
+        for module, group in modules.items()
+    ]
+
+
+def yabasanshiro(tree: RepoTree) -> list[Launcher]:
+    package = f"{STANDALONE}/yabasanshiro-sa"
+    script = Script(
+        f"{package}/scripts/start_yabasanshiro.sh",
+        tree.read(f"{package}/scripts/start_yabasanshiro.sh"),
+    )
+    option_line, option = script.find(r"=\$\(get_setting (\w+) ", "BIOS option")
+    test_line, test = script.find(r'if \[ "\$\w+" = (\w+) \]', "BIOS option test", option_line)
+    loop = next((lp for lp in script.loops() if lp.start > test_line), None)
+    if loop is None:
+        raise script.missing("loop over BIOS names")
+    find_line, find = script.find(
+        rf"\$\(find (\S+) -name \$\{{{loop.var}\}}", "find of each name", loop.start, loop.end
+    )
+    flag_line, flag = script.find(
+        r'="(-\w) \$\{\w+\}"', "flag the found path is passed with", find_line, loop.end
+    )
+    break_line, _ = script.find(r"^\s*break\b", "break on the first found", flag_line, loop.end)
+    return [
+        Launcher(
+            emulator="yabasanshiro",
+            build="standalone",
+            kind="names_searched",
+            names=loop.items,
+            source_ref=source_ref(
+                [(script.path, {option_line, test_line, *range(loop.start, loop.end + 1)})]
+            ),
+            case_insensitive=False,
+            search_dir=f"{find.group(1)} (recursive)",
+            passes_to=f"yabasanshiro {flag.group(1)} <first path found>",
+            fails_when_missing=False,
+            depends_on_option=f"{option.group(1)} = {test.group(1)} (game, then platform)",
+        )
+    ]
+
+
+def directory_records(
+    tree: RepoTree,
+    emulator: str,
+    package: str,
+    file: str,
+    folder_key: tuple[str, str],
+    file_keys: tuple[tuple[str, str], ...],
+) -> list[Launcher]:
+    """scans_dir records: the config hands a directory and leaves the file keys empty."""
+    configs = device_configs(tree, package, file)
+    name = posixpath.basename(file)
+
+    def read(path: str, text: str) -> tuple[object, set[int]]:
+        found = {(s.section, s.key): s for s in settings(text)}
+        if folder_key not in found:
+            raise LauncherPatternMissing(path, f"[{folder_key[0]}] {folder_key[1]}")
+        set_files = [k for k in file_keys if k in found and found[k].value]
+        if set_files:
+            raise LauncherPatternMissing(path, f"empty [{set_files[0][0]}] {set_files[0][1]}")
+        lines = {found[k].line for k in (folder_key, *file_keys) if k in found}
+        return found[folder_key].value, lines
+
+    unset = describe_keys("", file_keys).strip()
+    records = []
+    for device, folder, refs in per_device(tree, configs, f"{package}/package.mk", read):
+        passes = f"{name} [{folder_key[0]}] {folder_key[1]} = {folder}, {unset} unset"
+        records.append(
+            Launcher(
+                emulator=emulator,
+                build="standalone",
+                kind="scans_dir",
+                names=[],
+                source_ref=source_ref(refs),
+                search_dir=folder,
+                passes_to=passes,
+                device=device,
+            )
+        )
+    return records
+
+
+def duckstation(tree: RepoTree) -> list[Launcher]:
+    return directory_records(
+        tree,
+        "duckstation",
+        f"{STANDALONE}/duckstation-sa",
+        "settings.ini",
+        ("BIOS", "SearchDirectory"),
+        (("BIOS", "PathNTSCU"), ("BIOS", "PathNTSCJ"), ("BIOS", "PathPAL")),
+    )
+
+
+def aethersx2(tree: RepoTree) -> list[Launcher]:
+    return directory_records(
+        tree,
+        "aethersx2",
+        f"{STANDALONE}/aethersx2-sa",
+        "aethersx2/inis/PCSX2.ini",
+        ("Folders", "Bios"),
+        (("Filenames", "BIOS"),),
+    )
+
+
+def armsx2(tree: RepoTree) -> list[Launcher]:
+    return directory_records(
+        tree,
+        "armsx2",
+        f"{STANDALONE}/armsx2-sa",
+        "ARMSX2/inis/PCSX2.ini",
+        ("Folders", "Bios"),
+        (("Filenames", "BIOS"),),
+    )
+
+
+# Start scripts whose ln -s commands put a BIOS path where the emulator reads.
+LINK_SCRIPTS = (
+    ("drastic", f"{STANDALONE}/drastic-sa/scripts/start_drastic.sh"),
+    ("dolphin", f"{STANDALONE}/dolphin-sa/scripts/start_dolphin_gc.sh"),
+    ("flycast", f"{STANDALONE}/flycast-sa/scripts/start_flycast.sh"),
+    ("cemu", f"{STANDALONE}/cemu-sa/scripts/start_cemu.sh"),
+    ("rpcs3", f"{STANDALONE}/rpcs3-sa/scripts/start_rpcs3.sh"),
+)
+
+
+def links(tree: RepoTree) -> list[Launcher]:
+    records = []
+    for emulator, path in LINK_SCRIPTS:
+        script = Script(path, tree.read(path))
+        found = script.links(BIOS_ROOT)
+        if not found:
+            raise script.missing(f"ln -s of a path under {BIOS_ROOT}")
+        for link in found:
+            guard = _existence_guard(script, link)
+            if guard is not None:
+                link.lines.add(guard)
+            records.append(
+                link_record(
+                    emulator,
+                    script,
+                    link,
+                    case_insensitive=False if guard is not None else None,
+                    fails_when_missing=False,
+                )
+            )
+    return records
+
+
+def _existence_guard(script: Script, link: Link) -> int | None:
+    """The line of a -e test of the BIOS-side path the link waits for, if any."""
+    last = max(link.lines)
+    tests = [
+        number
+        for number, line in script.code()
+        if number < last
+        and re.search(
+            r"\[ -e \S*" + re.escape(link.bios_side) + "/", script.expand(line, number, set())
+        )
+    ]
+    return tests[-1] if tests and last - tests[-1] <= 3 else None
+
+
+EXTRACTORS: tuple[Callable[[RepoTree], list[Launcher]], ...] = (
+    retroarch,
+    amiberry,
+    hatari,
+    melonds,
+    dsperate,
+    nanoboyadvance,
+    xemu,
+    mednafen,
+    yabasanshiro,
+    duckstation,
+    aethersx2,
+    armsx2,
+    links,
+)
+
+
+def extract_launchers(tree: RepoTree) -> list[dict]:
+    """Every launcher record the extractors find, in extractor order."""
+    return [record.entry() for extractor in EXTRACTORS for record in extractor(tree)]

scripts/scraper/rocknix_scraper.py

@@ -24,6 +24,8 @@ try:
         fetch_github_latest_version,
         requirement_entry,
     )
+    from .rocknix_launchers import extract_launchers
+    from .shell_launchers import RepoTree
 except ImportError:
     from base_scraper import (
         BaseScraper,
@@ -31,6 +33,8 @@ except ImportError:
         fetch_github_latest_version,
         requirement_entry,
     )
+    from rocknix_launchers import extract_launchers
+    from shell_launchers import RepoTree
 
 PLATFORM_NAME = "rocknix"
 
@@ -53,6 +57,14 @@ STANDALONE_TREE_URL = (
 )
 
 
+def launcher_revision(url: str) -> str:
+    """The tag or commit a rocknix-systems URL is read at."""
+    found = re.search(r"/ROCKNIX/distribution/([^/]+)/projects/", url)
+    if found is None:
+        raise ValueError(f"{url} names no ROCKNIX/distribution revision")
+    return found.group(1)
+
+
 def standalone_emulators(package_names: list[str]) -> list[str]:
     """The standalone emulators ROCKNIX packages, by the names profiles use.
 
@@ -274,6 +286,10 @@ class Scraper(BaseScraper):
             # emulator as a RetroArch core.
             "dual_mode_cores": dual_mode_emulators(standalone, self._fetch_emulators_mk()),
             "systems": systems,
+            # Read at the revision the BIOS list comes from.
+            "launchers": extract_launchers(
+                RepoTree.github(GITHUB_REPO, launcher_revision(self.url))
+            ),
         }
 
 

scripts/scraper/shell_launchers.py

@@ -0,0 +1,531 @@
+"""Readers for launchers written as shell scripts and key=value configs.
+
+ROCKNIX and EmuDeck launch emulators from bash: a start script links or
+writes a BIOS path, a config shipped with the package names the file. These
+helpers read that text without running it. Each one targets a construct
+(an assignment, a for loop, an ln command, a key of a config) and the
+extractors built on them raise LauncherPatternMissing when the construct is
+gone, so a rewrite upstream stops the scrape instead of publishing nothing.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass, field
+
+try:
+    from .base_scraper import fetch_bytes, github_headers
+    from .batocera_launchers import LauncherShapeError
+except ImportError:
+    from base_scraper import fetch_bytes, github_headers
+    from batocera_launchers import LauncherShapeError
+
+NAME = r"[A-Za-z_][A-Za-z0-9_]*"
+# A single-quoted "$" is literal: it is carried as this mark through expansion.
+LITERAL_DOLLAR = "\x00"
+# A loop variable expands to this mark around its name, so the caller can
+# tell which part of a path the loop supplies.
+LOOP_MARK = "\x01"
+
+
+class LauncherPatternMissing(LauncherShapeError):
+    """A construct an extractor reads is absent from the launcher source."""
+
+    def __init__(self, path: str, what: str) -> None:
+        super().__init__(f"{path}: {what} not found")
+
+
+def span(first: int, last: int) -> str:
+    return str(first) if first == last else f"{first}-{last}"
+
+
+def merge_lines(lines: Iterable[int]) -> list[tuple[int, int]]:
+    """Line numbers as sorted spans, a gap of one line joined."""
+    spans: list[tuple[int, int]] = []
+    for line in sorted(set(lines)):
+        if spans and line <= spans[-1][1] + 2:
+            spans[-1] = (spans[-1][0], line)
+        else:
+            spans.append((line, line))
+    return spans
+
+
+def source_ref(parts: Iterable[tuple[str, Iterable[int]]]) -> str:
+    """path:a-b, c for each file, files in the order given."""
+    merged: dict[str, set[int]] = {}
+    for path, lines in parts:
+        merged.setdefault(path, set()).update(lines)
+    return ", ".join(
+        f"{path}:{', '.join(span(*s) for s in merge_lines(lines))}"
+        for path, lines in merged.items()
+    )
+
+
+@dataclass
+class Word:
+    """A shell word: as written, and with quotes removed."""
+
+    raw: str
+    text: str
+
+
+def words(line: str) -> list[Word]:
+    """The words of one command line, quotes removed, comments dropped.
+
+    Variables are left unexpanded, each written ${NAME} so the quoting that
+    ended a name ("$a""b") still ends it. A "$" inside single quotes is kept
+    as LITERAL_DOLLAR so a later expansion leaves it alone.
+    """
+    result: list[Word] = []
+    raw, text, quote, started, segment = "", "", "", False, 0
+
+    def close_segment() -> None:
+        nonlocal text, segment
+        text = text[:segment] + re.sub(rf"\$({NAME})", r"${\1}", text[segment:])
+        segment = len(text)
+
+    i = 0
+    while i < len(line):
+        char = line[i]
+        if quote == "'":
+            raw += char
+            if char == "'":
+                quote = ""
+                segment = len(text)
+            else:
+                text += LITERAL_DOLLAR if char == "$" else char
+        elif quote == '"':
+            raw += char
+            if char == "\\" and i + 1 < len(line) and line[i + 1] in '"\\$`':
+                raw += line[i + 1]
+                text += LITERAL_DOLLAR if line[i + 1] == "$" else line[i + 1]
+                i += 1
+            elif char == '"':
+                quote = ""
+                close_segment()
+            else:
+                text += char
+        elif char in "'\"":
+            close_segment()
+            quote, started = char, True
+            raw += char
+        elif char == "\\" and i + 1 < len(line):
+            raw += char + line[i + 1]
+            text += line[i + 1]
+            started = True
+            i += 1
+        elif char == "#" and not started:
+            break
+        elif char.isspace() or char == ";":
+            if started:
+                close_segment()
+                result.append(Word(raw, text))
+            raw, text, started, segment = "", "", False, 0
+            if char == ";":
+                result.append(Word(";", ";"))
+        else:
+            raw += char
+            text += char
+            started = True
+        i += 1
+    if started:
+        close_segment()
+        result.append(Word(raw, text))
+    return result
+
+
+@dataclass
+class Assignment:
+    line: int
+    value: str
+
+
+@dataclass
+class Loop:
+    """A for loop: its variable, its items and the lines they come from."""
+
+    var: str
+    items: list[str]
+    start: int
+    end: int
+    lines: set[int] = field(default_factory=set)
+
+
+@dataclass
+class Link:
+    """An ln -s whose link or target lies under the BIOS root."""
+
+    target: str
+    link: str
+    bios_side: str
+    names: list[str]
+    lines: set[int]
+    bios_is_target: bool = True
+    expands: dict[str, list[str]] = field(default_factory=dict)
+
+
+def _placeholders(text: str, marks: dict[str, Loop]) -> str:
+    """Loop marks written as {VAR}, the placeholder `expands` resolves."""
+    for mark, loop in marks.items():
+        text = text.replace(mark, "{" + loop.var + "}")
+    return text
+
+
+class Script:
+    """One shell script or makefile, read line by line."""
+
+    def __init__(self, path: str, text: str) -> None:
+        self.path = path
+        self.text = text
+        self.lines = text.splitlines()
+        self._assignments: dict[str, list[Assignment]] = {}
+        self._arrays: dict[str, tuple[int, list[str]]] = {}
+        for number, line in enumerate(self.lines, 1):
+            found = re.match(rf"\s*(?:local\s+|export\s+)?({NAME})=(.*)$", line)
+            if found is None:
+                continue
+            array = re.fullmatch(r"\((.*)\)\s*", found.group(2))
+            if array:
+                self._arrays[found.group(1)] = (number, [w.text for w in words(array.group(1))])
+                continue
+            value = words(found.group(2))
+            if value and value[0].raw:
+                self._assignments.setdefault(found.group(1), []).append(
+                    Assignment(number, value[0].text)
+                )
+
+    def missing(self, what: str) -> LauncherPatternMissing:
+        return LauncherPatternMissing(self.path, what)
+
+    def code(self) -> Iterable[tuple[int, str]]:
+        """Lines that are not comments."""
+        for number, line in enumerate(self.lines, 1):
+            if not line.lstrip().startswith("#"):
+                yield number, line
+
+    def find(
+        self, pattern: str, what: str, start: int = 1, end: int | None = None
+    ) -> tuple[int, re.Match[str]]:
+        """The first code line from start to end matching pattern."""
+        for number, line in self.code():
+            if number < start or (end is not None and number > end):
+                continue
+            found = re.search(pattern, line)
+            if found:
+                return number, found
+        raise self.missing(what)
+
+    def assignment_ending(self, suffix: str) -> Assignment | None:
+        """The first assignment whose value ends with suffix."""
+        return next(
+            (a for found in self._assignments.values() for a in found if a.value.endswith(suffix)),
+            None,
+        )
+
+    def value(self, name: str, before: int) -> Assignment | None:
+        """The last assignment of name above a line, else the first below it."""
+        assignments = self._assignments.get(name, [])
+        above = [a for a in assignments if a.line < before]
+        if above:
+            return above[-1]
+        return assignments[0] if assignments else None
+
+    def expand(
+        self, text: str, line: int, used: set[int], fixed: dict[str, str] | None = None
+    ) -> str:
+        """Variables replaced by their assigned value, unknown ones as ${NAME}."""
+        fixed = fixed or {}
+
+        def replace(found: re.Match[str], depth: int) -> str:
+            name = found.group(1) or found.group(2)
+            if name in fixed:
+                return fixed[name]
+            assignment = self.value(name, line)
+            if assignment is None or depth > 8:
+                return "${" + name + "}"
+            used.add(assignment.line)
+            return expand_once(assignment.value, assignment.line, depth + 1)
+
+        def expand_once(value: str, at: int, depth: int) -> str:
+            nonlocal line
+            saved, line = line, at
+            try:
+                return re.sub(rf"\$\{{({NAME})\}}|\$({NAME})", lambda m: replace(m, depth), value)
+            finally:
+                line = saved
+
+        return expand_once(text, line, 0).replace(LITERAL_DOLLAR, "$")
+
+    def loops(self) -> list[Loop]:
+        """Every for loop, items from a word list or from "${ARRAY[@]}"."""
+        found_loops = []
+        for number, line in self.code():
+            head = re.match(rf"\s*for ({NAME}) in (.*?)\s*(?:;\s*do)?\s*$", line)
+            if head is None:
+                continue
+            items = [w.text for w in words(head.group(2)) if w.text != ";"]
+            lines = {number}
+            if len(items) == 1 and (array := re.fullmatch(rf"\$\{{({NAME})\[@\]\}}", items[0])):
+                if array.group(1) not in self._arrays:
+                    raise self.missing(f"array {array.group(1)}")
+                array_line, items = self._arrays[array.group(1)]
+                lines.add(array_line)
+            found_loops.append(Loop(head.group(1), items, number, self._loop_end(number), lines))
+        return found_loops
+
+    def _loop_end(self, start: int) -> int:
+        depth = 0
+        for number, line in self.code():
+            if number < start:
+                continue
+            if re.match(r"\s*(for|while|until)\b", line):
+                depth += 1
+            if re.match(r"\s*done\b", line) or re.search(r";\s*done\s*$", line):
+                depth -= 1
+                if depth == 0:
+                    return number
+        raise self.missing(f"done of the loop at line {start}")
+
+    def links(self, bios_root: str, fixed: dict[str, str] | None = None) -> list[Link]:
+        """ln -s commands with the link or its target under bios_root.
+
+        Links in one loop, or written one by one as P/n linked to Q/n, are
+        one Link listing each n.
+        """
+        loops = self.loops()
+        found: list[Link] = []
+        for number, line in self.code():
+            command = words(line)
+            heads = [
+                i
+                for i, w in enumerate(command)
+                if w.raw == "ln"
+                and (i == 0 or command[i - 1].raw in {";", "then", "do", "&&", "||"})
+            ]
+            for head in heads:
+                args = []
+                for word in command[head + 1 :]:
+                    if word.raw in {";", "&&", "||"}:
+                        break
+                    args.append(word)
+                options = [w.text for w in args if w.text.startswith("-")]
+                operands = [w.text for w in args if not w.text.startswith("-")]
+                if not any("s" in o for o in options) or len(operands) != 2:
+                    continue
+                enclosing = [lp for lp in loops if lp.start < number <= lp.end]
+                env = dict(fixed or {})
+                marks = {f"{LOOP_MARK}{lp.var}{LOOP_MARK}": lp for lp in enclosing}
+                env.update({lp.var: mark for mark, lp in marks.items()})
+                used = {number}
+                target, link = (self.expand(op, number, used, env) for op in operands)
+                if not (_under(target, bios_root) or _under(link, bios_root)):
+                    continue
+                bios_is_target = _under(target, bios_root)
+                bios_side = (target if bios_is_target else link).rstrip("/")
+                folder, _, last = bios_side.rpartition("/")
+                present = [m for m in marks if m in target or m in link]
+                if any(m in folder for m in present):
+                    raise self.missing(
+                        f"loop variable only in the last segment of the ln at line {number}"
+                    )
+                names: list[str] = []
+                expands: dict[str, list[str]] = {}
+                if present:
+                    bios_side = folder
+                    if last in marks:
+                        names = list(marks[last].items)
+                    else:
+                        names = [_placeholders(last, marks)]
+                        expands = {lp.var: list(lp.items) for m, lp in marks.items() if m in last}
+                    if bios_is_target and last not in link:
+                        link = f"{link.rstrip('/')}/{last}"
+                    for mark in present:
+                        used |= marks[mark].lines
+                    template = _placeholders(last, marks) if expands else "<name>"
+                    target, link = (
+                        p.replace(last, template) if last in p else _placeholders(p, marks)
+                        for p in (target, link)
+                    )
+                duplicate = next((f for f in found if (f.target, f.link) == (target, link)), None)
+                if duplicate is not None:
+                    duplicate.lines |= used
+                    continue
+                found.append(Link(target, link, bios_side, names, used, bios_is_target, expands))
+        return _merge_links(found)
+
+
+def _under(path: str, root: str) -> bool:
+    return path.rstrip("/") == root.rstrip("/") or path.startswith(root.rstrip("/") + "/")
+
+
+def _merge_links(links: list[Link]) -> list[Link]:
+    """Single links P/n -> Q/n sharing P and Q folded into one, names in order."""
+    merged: list[Link] = []
+    folds: dict[tuple[str, str, bool], tuple[Link, list[Link]]] = {}
+    for link in links:
+        bios = link.bios_side.rstrip("/")
+        bios_is_target = link.bios_is_target
+        other = (link.link if bios_is_target else link.target).rstrip("/")
+        name = bios.rsplit("/", 1)[-1]
+        if link.names or other.rsplit("/", 1)[-1] != name:
+            merged.append(link)
+            continue
+        key = (bios.rsplit("/", 1)[0], other.rsplit("/", 1)[0], bios_is_target)
+        if key in folds:
+            folds[key][1].append(link)
+        else:
+            folds[key] = (link, [link])
+            merged.append(link)
+    folded: set[int] = set()
+    for (parent, other_parent, bios_is_target), (first, members) in folds.items():
+        if len(members) == 1:
+            continue
+        first.names = [m.bios_side.rstrip("/").rsplit("/", 1)[-1] for m in members]
+        first.bios_side = parent
+        bios_end, other_end = f"{parent}/<name>", f"{other_parent}/<name>"
+        first.target, first.link = (
+            (bios_end, other_end) if bios_is_target else (other_end, bios_end)
+        )
+        for member in members[1:]:
+            first.lines |= member.lines
+            folded.add(id(member))
+    return [link for link in merged if id(link) not in folded]
+
+
+@dataclass
+class Setting:
+    section: str
+    key: str
+    value: str
+    line: int
+
+
+def settings(text: str, separator: str = "=") -> list[Setting]:
+    """key = value lines under their [section], quotes around the value removed.
+
+    separator " " reads mednafen's "key value" form.
+    """
+    found, section = [], ""
+    for number, line in enumerate(text.splitlines(), 1):
+        stripped = line.strip()
+        if not stripped or stripped[0] in "#;":
+            continue
+        header = re.fullmatch(r"\[([^\]]+)\]", stripped)
+        if header:
+            section = header.group(1)
+            continue
+        if separator == " ":
+            pair = re.fullmatch(r"(\S+)(?:\s+(.*))?", stripped)
+        else:
+            pair = re.fullmatch(r"([^=\s]+)\s*=\s*(.*)", stripped)
+        if pair is None:
+            continue
+        value = (pair.group(2) or "").strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
+            value = value[1:-1]
+        found.append(Setting(section, pair.group(1), value, number))
+    return found
+
+
+def describe_keys(file: str, keys: tuple[tuple[str, str], ...]) -> str:
+    """file [section] key, key [other] key, sections named where they change."""
+    described, section = file, None
+    for index, (key_section, key) in enumerate(keys):
+        if key_section and key_section != section:
+            described += f" [{key_section}] {key}"
+            section = key_section
+        else:
+            described += (" " if index == 0 else ", ") + key
+    return described
+
+
+@dataclass
+class Entry:
+    """A directory entry: kind is "dir", "file" or "link"."""
+
+    kind: str
+    target: str = ""
+
+
+class RepoTree:
+    """Files of one repository revision, read on demand and kept."""
+
+    def __init__(
+        self,
+        read: Callable[[str], str],
+        listdir: Callable[[str], dict[str, Entry]],
+    ) -> None:
+        self._read = read
+        self._listdir = listdir
+        self._texts: dict[str, str] = {}
+        self._dirs: dict[str, dict[str, Entry]] = {}
+
+    def read(self, path: str) -> str:
+        if path not in self._texts:
+            self._texts[path] = self._read(path)
+        return self._texts[path]
+
+    def listdir(self, path: str) -> dict[str, Entry]:
+        if path not in self._dirs:
+            self._dirs[path] = self._listdir(path)
+        return self._dirs[path]
+
+    def has(self, path: str) -> bool:
+        parent, _, name = path.rpartition("/")
+        try:
+            return name in self.listdir(parent)
+        except LauncherPatternMissing:
+            return False
+
+    @classmethod
+    def from_files(cls, files: dict[str, str], links: dict[str, str] | None = None) -> RepoTree:
+        """A tree held in memory: file texts by path, symlink targets by path."""
+        links = links or {}
+
+        def read(path: str) -> str:
+            if path not in files:
+                raise LauncherPatternMissing(path, "file")
+            return files[path]
+
+        def listdir(path: str) -> dict[str, Entry]:
+            prefix = path.rstrip("/") + "/"
+            entries: dict[str, Entry] = {}
+            for name, target in links.items():
+                if name.startswith(prefix) and "/" not in name[len(prefix) :]:
+                    entries[name[len(prefix) :]] = Entry("link", target)
+            for name in files:
+                if name.startswith(prefix):
+                    rest = name[len(prefix) :]
+                    head, _, tail = rest.partition("/")
+                    entries.setdefault(head, Entry("dir" if tail else "file"))
+            if not entries:
+                raise LauncherPatternMissing(path, "directory")
+            return entries
+
+        return cls(read, listdir)
+
+    @classmethod
+    def github(cls, repo: str, rev: str) -> RepoTree:
+        """A GitHub repository at one revision: raw files and the trees API."""
+
+        def read(path: str) -> str:
+            url = f"https://raw.githubusercontent.com/{repo}/{rev}/{path}"
+            return fetch_bytes(url).decode("utf-8-sig")
+
+        def listdir(path: str) -> dict[str, Entry]:
+            url = f"https://api.github.com/repos/{repo}/git/trees/{rev}:{path}"
+            try:
+                listing = json.loads(fetch_bytes(url, headers=github_headers()))
+            except ConnectionError as e:
+                raise LauncherPatternMissing(path, f"directory ({e})") from e
+            entries = {}
+            for item in listing.get("tree", []):
+                if item["mode"] == "120000":
+                    entries[item["path"]] = Entry("link", read(f"{path}/{item['path']}").strip())
+                else:
+                    entries[item["path"]] = Entry("dir" if item["type"] == "tree" else "file")
+            return entries
+
+        return cls(read, listdir)