feat: read the amberelec launcher layer

amberelec_launchers reads runemu.py with ast and, at the commit the
scraper pins, the scripts its standalone table runs and the configs
their packages install: RetroArch and retrorun directories, the MAME
-rp path, atari800.cfg, Hatari's default TOS, amiberry's kickstarts
link and defaults, yabasanshiro, PICO-8, piemu, scummvm, freej2me and
easyrpg. 17 records.

The freej2me-lr.jar and pico8_64 notes now name the file in a sentence
about the emulator, as test_launcher_aliases asks once a launcher
records it.

scripts/scraper/amberelec_launchers.py

@@ -0,0 +1,429 @@
+"""AmberELEC's launcher layer, read from runemu.py and its emulator scripts.
+
+EmulationStation hands every launch to runemu.py. A libretro core runs in
+RetroArch or retrorun with the system directory their configs name, MAME
+machines get a .cmd file with a -rp search path, and every other emulator
+runs the script its standalone_emulators table names (/usr/bin/<name>.sh,
+installed from the package). Those scripts, and the configs their packages
+install, say which file under /storage/roms/bios is opened: yabasanshiro.sh
+tests saturn_bios.bin and passes it with -b, Hatari's default config names
+tos.img, the atari800 core is built to read a config naming its OS ROMs.
+
+runemu.py is read with ast, the scripts with shell_launchers' readers. An
+extractor whose construct is gone raises instead of returning nothing.
+"""
+
+from __future__ import annotations
+
+import ast
+import posixpath
+import re
+from collections.abc import Callable
+from typing import Any
+
+try:
+    from .configgen_launchers import Launcher, Source
+    from .shell_launchers import (
+        LauncherPatternMissing,
+        Script,
+        describe_keys,
+        settings,
+        source_ref,
+    )
+except ImportError:
+    from configgen_launchers import Launcher, Source  # type: ignore[no-redef]
+    from shell_launchers import (  # type: ignore[no-redef]
+        LauncherPatternMissing,
+        Script,
+        describe_keys,
+        settings,
+        source_ref,
+    )
+
+BIOS = "/storage/roms/bios"
+RUNEMU = "packages/amberelec/sources/scripts/runemu.py"
+RETROARCH_CFG = "packages/games/tools/retroarch/sources/retroarch.cfg"
+RETRORUN = "packages/games/tools/retrorun/retrorun.sh"
+HATARI_CONFIG = "packages/games/emulators/hatarisa/config/conf/Atari-ST-default.cfg"
+ATARI800_CONFIG = "packages/amberelec/config/distribution/configs/atari800.cfg"
+ATARI800_PATCH = "packages/games/libretro/atari800/patches/atari800-01-add-emuelec-platform.patch"
+AMIBERRY_PACKAGE = "packages/games/emulators/amiberry/package.mk"
+AMIBERRY_CONFIGS = "packages/games/emulators/amiberry/config/conf"
+FREEJ2ME = (
+    ("freej2me", "packages/games/libretro/freej2me/freej2me.sh"),
+    ("freej2me-plus", "packages/games/libretro/freej2me-plus/freej2me-plus.sh"),
+)
+EASYRPG = "packages/games/libretro/easyrpg/easyrpg.sh"
+
+# The script each standalone_emulators entry runs, as the package installs it
+# to /usr/bin. scripts() checks each against runemu's table.
+SCRIPTS = {
+    "AMIBERRY": "packages/games/emulators/amiberry/scripts/amiberry.sh",
+    "HATARISA": "packages/games/emulators/hatarisa/scripts/hatari.sh",
+    "SCUMMVMSA": "packages/games/emulators/scummvmsa/scummvm.sh",
+    "pico8": "packages/amberelec/sources/pico-8.sh",
+    "piemu": "packages/games/emulators/piemu/bin/piemu.sh",
+    "yabasanshiroSA": "packages/games/emulators/yabasanshiroSA/yabasanshiro.sh",
+}
+
+Read = Callable[[str], str]
+
+
+class Runemu:
+    """runemu.py, parsed: the standalone table and the RetroArch command."""
+
+    def __init__(self, read: Read) -> None:
+        self.src = Source(RUNEMU, read(RUNEMU))
+        node = self.src.assignment("standalone_emulators")
+        if not isinstance(node.value, ast.Dict):
+            self.src.fail("standalone_emulators is not a dict display")
+        self.table: dict[str, tuple[str, int]] = {}
+        for key, value in zip(node.value.keys, node.value.values):
+            if not (
+                isinstance(key, ast.Constant)
+                and isinstance(value, ast.Call)
+                and len(value.args) >= 2
+                and isinstance(value.args[1], ast.List)
+            ):
+                self.src.fail(f"entry at line {getattr(key, 'lineno', '?')} is not StandaloneEmulator(name, [args])")
+            args = [a.value for a in value.args[1].elts if isinstance(a, ast.Constant)]
+            script = next((a for a in args if str(a).endswith(".sh")), None)
+            if script is None:
+                self.src.fail(f"{ast.unparse(key)} runs no .sh script")
+            self.table[str(key.value)] = (str(script), value.lineno)
+
+    def script(self, emulator: str) -> tuple[str, int]:
+        """The repository script runemu runs for emulator, and the table line."""
+        if emulator not in self.table:
+            self.src.fail(f"standalone_emulators has no {emulator}")
+        installed, line = self.table[emulator]
+        path = SCRIPTS[emulator]
+        if posixpath.basename(installed) != posixpath.basename(path):
+            self.src.fail(f"{emulator} runs {installed}, not {posixpath.basename(path)}")
+        return path, line
+
+
+def retroarch(read: Read) -> list[Launcher]:
+    """system_directory of the retroarch.cfg the retroarch package installs."""
+    found = [s for s in settings(read(RETROARCH_CFG)) if s.key == "system_directory"]
+    if len(found) != 1:
+        raise LauncherPatternMissing(RETROARCH_CFG, "one system_directory")
+    return [Launcher(
+        emulator="retroarch", build="libretro", kind="passes_path", names=[],
+        search_dir=found[0].value, passes_to="retroarch.cfg system_directory",
+        fails_when_missing=False, source_ref=source_ref([(RETROARCH_CFG, [found[0].line])]),
+    )]
+
+
+def retrorun(read: Read) -> list[Launcher]:
+    """The -d directory retrorun.sh passes to retrorun."""
+    script = Script(RETRORUN, read(RETRORUN))
+    line, found = script.find(r"^\s*/usr/bin/retrorun\s.*\s-d\s+(\S+)", "retrorun -d")
+    return [Launcher(
+        emulator="retrorun", build="libretro", kind="passes_path", names=[],
+        search_dir=found.group(1).strip('"'), passes_to="command line -d",
+        fails_when_missing=False, source_ref=source_ref([(RETRORUN, [line])]),
+    )]
+
+
+def mame(runemu: Runemu) -> list[Launcher]:
+    """The -rp search path of the .cmd file runemu writes for a MAME machine."""
+    src = runemu.src
+    commands = [
+        node for node in ast.walk(src.function("get_retroarch_command"))
+        if isinstance(node, ast.JoinedStr)
+        and any(isinstance(p, ast.Constant) and " -rp " in str(p.value) for p in node.values)
+    ]
+    if not commands:
+        src.fail("get_retroarch_command writes no -rp search path")
+    paths = []
+    for node in commands:
+        text = "".join(str(p.value) for p in node.values if isinstance(p, ast.Constant))
+        paths.append(text.split(" -rp ", 1)[1].split()[0])
+    if len(set(paths)) != 1:
+        src.fail(f"the MAME commands differ in their search path: {sorted(set(paths))}")
+    guard = next(
+        node for node in ast.walk(src.function("get_retroarch_command"))
+        if isinstance(node, ast.If) and ast.unparse(node.test) == "self.emulator == 'mame'"
+    )
+    return [Launcher(
+        emulator="mame", build="libretro", kind="passes_path", names=[],
+        search_dir=paths[0], passes_to=".cmd -rp", fails_when_missing=False,
+        depends_on_option=ast.unparse(guard.test),
+        source_ref=src.ref([
+            src.span(guard.test), *((n.lineno, n.end_lineno or n.lineno) for n in commands),
+        ]),
+    )]
+
+
+def _bios_files(path: str, text: str) -> tuple[list[tuple[tuple[str, str], str]], set[int]]:
+    """((section, key), name under BIOS) for each config key whose value is a BIOS file."""
+    prefix = BIOS + "/"
+    found = [s for s in settings(text) if s.value.startswith(prefix)]
+    if not found:
+        raise LauncherPatternMissing(path, f"key naming a file under {BIOS}")
+    return [((s.section, s.key), s.value[len(prefix):]) for s in found], {s.line for s in found}
+
+
+def hatari(read: Read, runemu: Runemu) -> list[Launcher]:
+    """szTosImageFileName of the default config hatari.sh copies and passes."""
+    path, table_line = runemu.script("HATARISA")
+    script = Script(path, read(path))
+    name = posixpath.basename(HATARI_CONFIG)
+    copied, _ = script.find(rf"^\s*cp\s+\S*/{re.escape(name)}\s", f"cp of {name}")
+    passed, _ = script.find(r"\shatari\s+--configfile\s", "hatari --configfile")
+    files, lines = _bios_files(HATARI_CONFIG, read(HATARI_CONFIG))
+    return [Launcher(
+        emulator="HATARISA", build="standalone", kind="fixed_name",
+        names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS,
+        passes_to=describe_keys(name, tuple(k for k, _n in files)),
+        fails_when_missing=False,
+        source_ref=source_ref([
+            (RUNEMU, [table_line]), (path, [copied, passed]), (HATARI_CONFIG, lines),
+        ]),
+    )]
+
+
+def atari800(read: Read) -> list[Launcher]:
+    """The ROM keys of the config the atari800 core is built to read."""
+    patch = Script(ATARI800_PATCH, read(ATARI800_PATCH))
+    line, found = patch.find(r'DEFAULT_CFG_NAME="\\"([^"\\]+)\\""', "DEFAULT_CFG_NAME")
+    built = found.group(1)
+    if not built.endswith("/" + ATARI800_CONFIG.split("/config/", 1)[1]):
+        raise LauncherPatternMissing(ATARI800_PATCH, f"DEFAULT_CFG_NAME naming {ATARI800_CONFIG}")
+    files, lines = _bios_files(ATARI800_CONFIG, read(ATARI800_CONFIG))
+    return [Launcher(
+        emulator="atari800", build="libretro", kind="fixed_name",
+        names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS,
+        passes_to=describe_keys(posixpath.basename(ATARI800_CONFIG), tuple(k for k, _n in files)),
+        fails_when_missing=False,
+        source_ref=source_ref([(ATARI800_PATCH, [line]), (ATARI800_CONFIG, lines)]),
+    )]
+
+
+def yabasanshiro(read: Read, runemu: Runemu) -> list[Launcher]:
+    """`if [ -e <bios> ]; then BIOS="-b <bios>"`, skipped when HLE is chosen."""
+    path, table_line = runemu.script("yabasanshiroSA")
+    script = Script(path, read(path))
+    tested, test = script.find(rf'if \[ -e "({re.escape(BIOS)}/[^"]+)" \]', "BIOS presence test")
+    flag_line, flag = script.find(
+        rf'BIOS="(-\w+) {re.escape(test.group(1))}"', "BIOS flag", tested
+    )
+    hle_line, hle = script.find(r'get_setting "(\w*hle\w*)"', "HLE setting")
+    guard_line, _ = script.find(r'if \[ "\$\{HLEBIOS\}" != 1 \]', "HLE test")
+    used_line, _ = script.find(r"\$\{BIOS\}", "BIOS on the command line", flag_line + 1)
+    return [Launcher(
+        emulator="yabasanshiroSA", build="standalone", kind="fixed_name",
+        names=[test.group(1)[len(BIOS) + 1:]], case_insensitive=False, search_dir=BIOS,
+        passes_to=f"command line {flag.group(1)}", fails_when_missing=False,
+        depends_on_option=hle.group(1),
+        source_ref=source_ref([
+            (RUNEMU, [table_line]),
+            (path, [hle_line, guard_line, tested, flag_line, used_line]),
+        ]),
+    )]
+
+
+def pico8(read: Read, runemu: Runemu) -> list[Launcher]:
+    """`if [ ! -f A ] || [ ! -f B ]; then text_viewer ...; else A ...`."""
+    path, table_line = runemu.script("pico8")
+    script = Script(path, read(path))
+    line, test = script.find(r'if \[ ! -f "([^"]+)" \] \|\| \[ ! -f "([^"]+)" \]', "binary test")
+    message_line, message = script.find(r'text_viewer .*-t "([^"]+)"', "missing message", line)
+    binary = test.group(1)
+    run_line, _ = script.find(rf"^\s*{re.escape(binary)}\s", f"{binary} run", message_line)
+    folders = {posixpath.dirname(test.group(1)), posixpath.dirname(test.group(2))}
+    if len(folders) != 1:
+        raise script.missing("both binaries in one folder")
+    return [Launcher(
+        emulator="pico8", build="standalone", kind="fixed_name",
+        names=[posixpath.basename(test.group(1)), posixpath.basename(test.group(2))],
+        case_insensitive=False, search_dir=folders.pop(), passes_to="exec",
+        fails_when_missing=message.group(1),
+        source_ref=source_ref([(RUNEMU, [table_line]), (path, [line, message_line, run_line])]),
+    )]
+
+
+def piemu(read: Read, runemu: Runemu) -> list[Launcher]:
+    """The BIOS file piemu.sh hands to mkpfi to build the flash image."""
+    path, table_line = runemu.script("piemu")
+    script = Script(path, read(path))
+    names: list[str] = []
+    lines: set[int] = set()
+    for number, line in script.code():
+        found = re.search(r'/usr/bin/mkpfi\s+"([^"]+)"', line)
+        if found is None:
+            continue
+        used: set[int] = {number}
+        value = script.expand(found.group(1), number, used)
+        if value.startswith(BIOS + "/") and value[len(BIOS) + 1:] not in names:
+            names.append(value[len(BIOS) + 1:])
+        lines |= used
+    if not names:
+        raise script.missing(f"mkpfi of a file under {BIOS}")
+    return [Launcher(
+        emulator="piemu", build="standalone", kind="fixed_name", names=names,
+        case_insensitive=False, search_dir=BIOS, passes_to="mkpfi argument",
+        fails_when_missing=False,
+        source_ref=source_ref([(RUNEMU, [table_line]), (path, lines)]),
+    )]
+
+
+def amiberry(read: Read, runemu: Runemu) -> list[Launcher]:
+    """The kickstarts link of the package, and the ROM each default config names."""
+    package = Script(AMIBERRY_PACKAGE, read(AMIBERRY_PACKAGE))
+    links = [link for link in package.links(BIOS) if link.bios_is_target]
+    if len(links) != 1 or links[0].bios_side.rstrip("/") != BIOS:
+        raise package.missing(f"ln -s {BIOS} to the kickstarts folder")
+    link = links[0]
+    installed = re.sub(r"^\$\{INSTALL\}", "", link.link)
+    path, table_line = runemu.script("AMIBERRY")
+    script = Script(path, read(path))
+    launchers = [Launcher(
+        emulator="AMIBERRY", build="standalone", kind="links", names=[],
+        case_insensitive=False, search_dir=BIOS,
+        passes_to=f"{installed} is a symlink to it", fails_when_missing=False,
+        source_ref=source_ref([(AMIBERRY_PACKAGE, link.lines)]),
+    )]
+    for number, line in script.code():
+        copied = re.search(r"cp\s+\$AMIBERRY_CONFIG_DIR/(\S+\.uae)\s", line)
+        if copied is None:
+            continue
+        config = f"{AMIBERRY_CONFIGS}/{copied.group(1)}"
+        found = [
+            s for s in settings(read(config))
+            if s.key == "kickstart_rom_file" and s.value.startswith("$(FILE_PATH)/")
+        ]
+        rom_path = [s for s in settings(read(config)) if s.key == "amiberry.rom_path"]
+        if len(found) != 1 or len(rom_path) != 1:
+            raise LauncherPatternMissing(config, "kickstart_rom_file under $(FILE_PATH)")
+        if rom_path[0].value.rstrip("/").rsplit("/", 1)[-1] != installed.rstrip("/").rsplit("/", 1)[-1]:
+            raise LauncherPatternMissing(config, f"amiberry.rom_path naming {installed}")
+        condition_line = max(
+            n for n, text in script.code()
+            if n < number and re.match(r"\s*(if|else)\b", text)
+        )
+        branch = script.lines[condition_line - 1].strip()
+        if branch == "else":
+            opened = max(
+                n for n, text in script.code()
+                if n < condition_line and re.match(r"\s*if\b", text)
+            )
+            test = _shell_test(script, opened)
+            condition = f"not ({test})"
+        else:
+            test = _shell_test(script, condition_line)
+            condition = test
+        launchers.append(Launcher(
+            emulator="AMIBERRY", build="standalone", kind="fixed_name",
+            slot=copied.group(1).removesuffix("-default.uae").removeprefix("Amiga"),
+            names=[found[0].value.removeprefix("$(FILE_PATH)/")], case_insensitive=False,
+            search_dir=BIOS,
+            passes_to=f"{copied.group(1)} kickstart_rom_file, $(FILE_PATH) = {rom_path[0].value}",
+            fails_when_missing=False,
+            depends_on_option=condition,
+            source_ref=source_ref([
+                (RUNEMU, [table_line]), (path, [condition_line, number]),
+                (config, [rom_path[0].line, found[0].line]),
+            ]),
+        ))
+    if len(launchers) == 1:
+        raise script.missing("cp of a default .uae config")
+    return launchers
+
+
+def _shell_test(script: Script, line: int) -> str:
+    """The test of `if <test>; then` at a line."""
+    found = re.match(r"\s*if\s+(.*?);\s*then\s*$", script.lines[line - 1])
+    if found is None:
+        raise script.missing(f"if ...; then at line {line}")
+    return found.group(1)
+
+
+def scummvm(read: Read) -> list[Launcher]:
+    """The paths scummvm.sh writes into the libretro core's scummvm.ini."""
+    path = SCRIPTS["SCUMMVMSA"]
+    script = Script(path, read(path))
+    branch, _ = script.find(r'^\s*"libretro"\)', "libretro branch")
+    launchers = []
+    for number, line in script.code():
+        if number < branch:
+            continue
+        written = re.search(r"s/\^/(\w*path)=((?:\\/[^\\/n]+)+)\\n/", line)
+        if written is None:
+            continue
+        folder = written.group(2).replace("\\/", "/")
+        if "/roms/bios/" not in folder + "/" or written.group(1) == "browser_lastpath":
+            continue
+        launchers.append(Launcher(
+            emulator="scummvm", build="libretro", kind="passes_path", names=[],
+            search_dir=folder, passes_to=f"scummvm.ini {written.group(1)}",
+            fails_when_missing=False, depends_on_option="scummvm.sh libretro",
+            source_ref=source_ref([(path, [branch, number])]),
+        ))
+    if not launchers:
+        raise script.missing("a scummvm.ini path written under /roms/bios")
+    return launchers
+
+
+def freej2me(read: Read) -> list[Launcher]:
+    """`if [ ! -f <bios>/<jar> ]; then cp <shipped jar> <bios>`."""
+    launchers = []
+    for core, path in FREEJ2ME:
+        script = Script(path, read(path))
+        tested, test = script.find(
+            rf'if \[ ! -f "{re.escape(BIOS)}/([^"]+)" \]', "jar presence test"
+        )
+        jar = test.group(1)
+        lines = [tested]
+        source = None
+        for number, line in script.code():
+            copied = re.match(rf"\s*cp\s+(?:-\S+\s+)*(\S+/{re.escape(jar)})\s+{re.escape(BIOS)}/?\s*$", line)
+            if copied:
+                lines.append(number)
+                source = copied.group(1)
+        if source is None:
+            raise script.missing(f"cp of {jar} into {BIOS}")
+        launchers.append(Launcher(
+            emulator=core, build="libretro", kind="copies", names=[jar],
+            case_insensitive=False, search_dir=posixpath.dirname(source), passes_to=BIOS,
+            fails_when_missing=False, source_ref=source_ref([(path, lines)]),
+        ))
+    return launchers
+
+
+def easyrpg(read: Read) -> list[Launcher]:
+    """Each install_bios call: a runtime package downloaded and unpacked under BIOS."""
+    script = Script(EASYRPG, read(EASYRPG))
+    url_line, url = script.find(r'curl -Lo "\$\{BIOS\}" "([^"]+)"', "curl of the package")
+    launchers = []
+    for number, line in script.code():
+        call = re.match(r"\s*install_bios\s+(\S+)\s+(\S+)\s*$", line)
+        if call is None:
+            continue
+        used: set[int] = {number, url_line}
+        source = script.expand(url.group(1), number, used, {"BIOS": call.group(1)})
+        install = script.value("INSTALLPATH", number)
+        if install is None or install.value.rstrip("/") != BIOS:
+            raise script.missing(f"INSTALLPATH {BIOS}")
+        used.add(install.line)
+        launchers.append(Launcher(
+            emulator="easyrpg", build="libretro", kind="copies", names=[],
+            search_dir=source, passes_to=f"{BIOS}/{call.group(2)}",
+            fails_when_missing=False, source_ref=source_ref([(EASYRPG, used)]),
+        ))
+    if not launchers:
+        raise script.missing("install_bios call")
+    return launchers
+
+
+def extract_launchers(read: Read) -> list[dict[str, Any]]:
+    """Every launcher record, read through `read(repo-relative path)`."""
+    runemu = Runemu(read)
+    launchers = [
+        *retroarch(read), *retrorun(read), *mame(runemu),
+        *atari800(read), *scummvm(read), *freej2me(read), *easyrpg(read),
+        *amiberry(read, runemu), *hatari(read, runemu), *pico8(read, runemu),
+        *piemu(read, runemu), *yabasanshiro(read, runemu),
+    ]
+    return [launcher.entry() for launcher in launchers]

scripts/scraper/amberelec_scraper.py

@@ -23,6 +23,9 @@ Verification (batocera-systems at that commit):
   (packages/amberelec/profile.d/99-distribution.conf:320) passes
   `--strictfilter`, which the script's argument switch (lines 665-687) does
   not handle: that call prints nothing and exits 0.
+
+The launcher layer is read at the same commit (amberelec_launchers): runemu.py,
+the scripts its standalone table runs and the configs their packages install.
 """
 
 from __future__ import annotations
@@ -32,6 +35,7 @@ import re
 import urllib.error
 import urllib.request
 
+from .amberelec_launchers import extract_launchers
 from .base_scraper import BaseScraper, BiosRequirement, github_headers
 from .batocera_scraper import (
     SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
@@ -155,6 +159,9 @@ class Scraper(BaseScraper):
             "standalone_cores": standalone,
             "dual_mode_cores": dual,
             "systems": systems,
+            "launchers": extract_launchers(
+                lambda path: self._fetch_text(_RAW.format(ref=self.commit) + path)
+            ),
         }