chore: merge integ into fix-launch-names

scripts/scraper/amberelec_launchers.py

@@ -0,0 +1,429 @@
+"""AmberELEC's launcher layer, read from runemu.py and its emulator scripts.
+
+EmulationStation hands every launch to runemu.py. A libretro core runs in
+RetroArch or retrorun with the system directory their configs name, MAME
+machines get a .cmd file with a `-rp` search path, and every other emulator
+runs the script its standalone_emulators table names (/usr/bin/<name>.sh,
+installed from the package). Those scripts, and the configs their packages
+install, say which file under /storage/roms/bios is opened: yabasanshiro.sh
+tests saturn_bios.bin and passes it with -b, Hatari's default config names
+tos.img, the atari800 core is built to read a config naming its OS ROMs.
+
+runemu.py is read with ast, the scripts with shell_launchers' readers. An
+extractor whose construct is gone raises instead of returning nothing.
+"""
+
+from __future__ import annotations
+
+import ast
+import posixpath
+import re
+from collections.abc import Callable
+from typing import Any
+
+try:
+    from .configgen_launchers import Launcher, Source
+    from .shell_launchers import (
+        LauncherPatternMissing,
+        Script,
+        describe_keys,
+        settings,
+        source_ref,
+    )
+except ImportError:
+    from configgen_launchers import Launcher, Source  # type: ignore[no-redef]
+    from shell_launchers import (  # type: ignore[no-redef]
+        LauncherPatternMissing,
+        Script,
+        describe_keys,
+        settings,
+        source_ref,
+    )
+
+BIOS = "/storage/roms/bios"
+RUNEMU = "packages/amberelec/sources/scripts/runemu.py"
+RETROARCH_CFG = "packages/games/tools/retroarch/sources/retroarch.cfg"
+RETRORUN = "packages/games/tools/retrorun/retrorun.sh"
+HATARI_CONFIG = "packages/games/emulators/hatarisa/config/conf/Atari-ST-default.cfg"
+ATARI800_CONFIG = "packages/amberelec/config/distribution/configs/atari800.cfg"
+ATARI800_PATCH = "packages/games/libretro/atari800/patches/atari800-01-add-emuelec-platform.patch"
+AMIBERRY_PACKAGE = "packages/games/emulators/amiberry/package.mk"
+AMIBERRY_CONFIGS = "packages/games/emulators/amiberry/config/conf"
+FREEJ2ME = (
+    ("freej2me", "packages/games/libretro/freej2me/freej2me.sh"),
+    ("freej2me-plus", "packages/games/libretro/freej2me-plus/freej2me-plus.sh"),
+)
+EASYRPG = "packages/games/libretro/easyrpg/easyrpg.sh"
+
+# The script each standalone_emulators entry runs, as the package installs it
+# to /usr/bin. scripts() checks each against runemu's table.
+SCRIPTS = {
+    "AMIBERRY": "packages/games/emulators/amiberry/scripts/amiberry.sh",
+    "HATARISA": "packages/games/emulators/hatarisa/scripts/hatari.sh",
+    "SCUMMVMSA": "packages/games/emulators/scummvmsa/scummvm.sh",
+    "pico8": "packages/amberelec/sources/pico-8.sh",
+    "piemu": "packages/games/emulators/piemu/bin/piemu.sh",
+    "yabasanshiroSA": "packages/games/emulators/yabasanshiroSA/yabasanshiro.sh",
+}
+
+Read = Callable[[str], str]
+
+
+class Runemu:
+    """runemu.py, parsed: the standalone table and the RetroArch command."""
+
+    def __init__(self, read: Read) -> None:
+        self.src = Source(RUNEMU, read(RUNEMU))
+        node = self.src.assignment("standalone_emulators")
+        if not isinstance(node.value, ast.Dict):
+            self.src.fail("standalone_emulators is not a dict display")
+        self.table: dict[str, tuple[str, int]] = {}
+        for key, value in zip(node.value.keys, node.value.values):
+            if not (
+                isinstance(key, ast.Constant)
+                and isinstance(value, ast.Call)
+                and len(value.args) >= 2
+                and isinstance(value.args[1], ast.List)
+            ):
+                self.src.fail(f"entry at line {getattr(key, 'lineno', '?')} is not StandaloneEmulator(name, [args])")
+            args = [a.value for a in value.args[1].elts if isinstance(a, ast.Constant)]
+            script = next((a for a in args if str(a).endswith(".sh")), None)
+            if script is None:
+                self.src.fail(f"{ast.unparse(key)} runs no .sh script")
+            self.table[str(key.value)] = (str(script), value.lineno)
+
+    def script(self, emulator: str) -> tuple[str, int]:
+        """The repository script runemu runs for emulator, and the table line."""
+        if emulator not in self.table:
+            self.src.fail(f"standalone_emulators has no {emulator}")
+        installed, line = self.table[emulator]
+        path = SCRIPTS[emulator]
+        if posixpath.basename(installed) != posixpath.basename(path):
+            self.src.fail(f"{emulator} runs {installed}, not {posixpath.basename(path)}")
+        return path, line
+
+
+def retroarch(read: Read) -> list[Launcher]:
+    """system_directory of the retroarch.cfg the retroarch package installs."""
+    found = [s for s in settings(read(RETROARCH_CFG)) if s.key == "system_directory"]
+    if len(found) != 1:
+        raise LauncherPatternMissing(RETROARCH_CFG, "one system_directory")
+    return [Launcher(
+        emulator="retroarch", build="libretro", kind="passes_path", names=[],
+        search_dir=found[0].value, passes_to="retroarch.cfg system_directory",
+        fails_when_missing=False, source_ref=source_ref([(RETROARCH_CFG, [found[0].line])]),
+    )]
+
+
+def retrorun(read: Read) -> list[Launcher]:
+    """The -d directory retrorun.sh passes to retrorun."""
+    script = Script(RETRORUN, read(RETRORUN))
+    line, found = script.find(r"^\s*/usr/bin/retrorun\s.*\s-d\s+(\S+)", "retrorun -d")
+    return [Launcher(
+        emulator="retrorun", build="libretro", kind="passes_path", names=[],
+        search_dir=found.group(1).strip('"'), passes_to="command line -d",
+        fails_when_missing=False, source_ref=source_ref([(RETRORUN, [line])]),
+    )]
+
+
+def mame(runemu: Runemu) -> list[Launcher]:
+    """The `-rp` search path of the .cmd file runemu writes for a MAME machine."""
+    src = runemu.src
+    commands = [
+        node for node in ast.walk(src.function("get_retroarch_command"))
+        if isinstance(node, ast.JoinedStr)
+        and any(isinstance(p, ast.Constant) and " -rp " in str(p.value) for p in node.values)
+    ]
+    if not commands:
+        src.fail("get_retroarch_command writes no -rp search path")
+    paths = []
+    for node in commands:
+        text = "".join(str(p.value) for p in node.values if isinstance(p, ast.Constant))
+        paths.append(text.split(" -rp ", 1)[1].split()[0])
+    if len(set(paths)) != 1:
+        src.fail(f"the MAME commands differ in their search path: {sorted(set(paths))}")
+    guard = next(
+        node for node in ast.walk(src.function("get_retroarch_command"))
+        if isinstance(node, ast.If) and ast.unparse(node.test) == "self.emulator == 'mame'"
+    )
+    return [Launcher(
+        emulator="mame", build="libretro", kind="passes_path", names=[],
+        search_dir=paths[0], passes_to=".cmd -rp", fails_when_missing=False,
+        depends_on_option=ast.unparse(guard.test),
+        source_ref=src.ref([
+            src.span(guard.test), *((n.lineno, n.end_lineno or n.lineno) for n in commands),
+        ]),
+    )]
+
+
+def _bios_files(path: str, text: str) -> tuple[list[tuple[tuple[str, str], str]], set[int]]:
+    """((section, key), name under BIOS) for each config key whose value is a BIOS file."""
+    prefix = BIOS + "/"
+    found = [s for s in settings(text) if s.value.startswith(prefix)]
+    if not found:
+        raise LauncherPatternMissing(path, f"key naming a file under {BIOS}")
+    return [((s.section, s.key), s.value[len(prefix):]) for s in found], {s.line for s in found}
+
+
+def hatari(read: Read, runemu: Runemu) -> list[Launcher]:
+    """szTosImageFileName of the default config hatari.sh copies and passes."""
+    path, table_line = runemu.script("HATARISA")
+    script = Script(path, read(path))
+    name = posixpath.basename(HATARI_CONFIG)
+    copied, _ = script.find(rf"^\s*cp\s+\S*/{re.escape(name)}\s", f"cp of {name}")
+    passed, _ = script.find(r"\shatari\s+--configfile\s", "hatari --configfile")
+    files, lines = _bios_files(HATARI_CONFIG, read(HATARI_CONFIG))
+    return [Launcher(
+        emulator="HATARISA", build="standalone", kind="fixed_name",
+        names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS,
+        passes_to=describe_keys(name, tuple(k for k, _n in files)),
+        fails_when_missing=False,
+        source_ref=source_ref([
+            (RUNEMU, [table_line]), (path, [copied, passed]), (HATARI_CONFIG, lines),
+        ]),
+    )]
+
+
+def atari800(read: Read) -> list[Launcher]:
+    """The ROM keys of the config the atari800 core is built to read."""
+    patch = Script(ATARI800_PATCH, read(ATARI800_PATCH))
+    line, found = patch.find(r'DEFAULT_CFG_NAME="\\"([^"\\]+)\\""', "DEFAULT_CFG_NAME")
+    built = found.group(1)
+    if not built.endswith("/" + ATARI800_CONFIG.split("/config/", 1)[1]):
+        raise LauncherPatternMissing(ATARI800_PATCH, f"DEFAULT_CFG_NAME naming {ATARI800_CONFIG}")
+    files, lines = _bios_files(ATARI800_CONFIG, read(ATARI800_CONFIG))
+    return [Launcher(
+        emulator="atari800", build="libretro", kind="fixed_name",
+        names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS,
+        passes_to=describe_keys(posixpath.basename(ATARI800_CONFIG), tuple(k for k, _n in files)),
+        fails_when_missing=False,
+        source_ref=source_ref([(ATARI800_PATCH, [line]), (ATARI800_CONFIG, lines)]),
+    )]
+
+
+def yabasanshiro(read: Read, runemu: Runemu) -> list[Launcher]:
+    """`if [ -e <bios> ]; then BIOS="-b <bios>"`, skipped when HLE is chosen."""
+    path, table_line = runemu.script("yabasanshiroSA")
+    script = Script(path, read(path))
+    tested, test = script.find(rf'if \[ -e "({re.escape(BIOS)}/[^"]+)" \]', "BIOS presence test")
+    flag_line, flag = script.find(
+        rf'BIOS="(-\w+) {re.escape(test.group(1))}"', "BIOS flag", tested
+    )
+    hle_line, hle = script.find(r'get_setting "(\w*hle\w*)"', "HLE setting")
+    guard_line, _ = script.find(r'if \[ "\$\{HLEBIOS\}" != 1 \]', "HLE test")
+    used_line, _ = script.find(r"\$\{BIOS\}", "BIOS on the command line", flag_line + 1)
+    return [Launcher(
+        emulator="yabasanshiroSA", build="standalone", kind="fixed_name",
+        names=[test.group(1)[len(BIOS) + 1:]], case_insensitive=False, search_dir=BIOS,
+        passes_to=f"command line {flag.group(1)}", fails_when_missing=False,
+        depends_on_option=hle.group(1),
+        source_ref=source_ref([
+            (RUNEMU, [table_line]),
+            (path, [hle_line, guard_line, tested, flag_line, used_line]),
+        ]),
+    )]
+
+
+def pico8(read: Read, runemu: Runemu) -> list[Launcher]:
+    """`if [ ! -f A ] || [ ! -f B ]; then text_viewer ...; else A ...`."""
+    path, table_line = runemu.script("pico8")
+    script = Script(path, read(path))
+    line, test = script.find(r'if \[ ! -f "([^"]+)" \] \|\| \[ ! -f "([^"]+)" \]', "binary test")
+    message_line, message = script.find(r'text_viewer .*-t "([^"]+)"', "missing message", line)
+    binary = test.group(1)
+    run_line, _ = script.find(rf"^\s*{re.escape(binary)}\s", f"{binary} run", message_line)
+    folders = {posixpath.dirname(test.group(1)), posixpath.dirname(test.group(2))}
+    if len(folders) != 1:
+        raise script.missing("both binaries in one folder")
+    return [Launcher(
+        emulator="pico8", build="standalone", kind="fixed_name",
+        names=[posixpath.basename(test.group(1)), posixpath.basename(test.group(2))],
+        case_insensitive=False, search_dir=folders.pop(), passes_to="exec",
+        fails_when_missing=message.group(1),
+        source_ref=source_ref([(RUNEMU, [table_line]), (path, [line, message_line, run_line])]),
+    )]
+
+
+def piemu(read: Read, runemu: Runemu) -> list[Launcher]:
+    """The BIOS file piemu.sh hands to mkpfi to build the flash image."""
+    path, table_line = runemu.script("piemu")
+    script = Script(path, read(path))
+    names: list[str] = []
+    lines: set[int] = set()
+    for number, line in script.code():
+        found = re.search(r'/usr/bin/mkpfi\s+"([^"]+)"', line)
+        if found is None:
+            continue
+        used: set[int] = {number}
+        value = script.expand(found.group(1), number, used)
+        if value.startswith(BIOS + "/") and value[len(BIOS) + 1:] not in names:
+            names.append(value[len(BIOS) + 1:])
+        lines |= used
+    if not names:
+        raise script.missing(f"mkpfi of a file under {BIOS}")
+    return [Launcher(
+        emulator="piemu", build="standalone", kind="fixed_name", names=names,
+        case_insensitive=False, search_dir=BIOS, passes_to="mkpfi argument",
+        fails_when_missing=False,
+        source_ref=source_ref([(RUNEMU, [table_line]), (path, lines)]),
+    )]
+
+
+def amiberry(read: Read, runemu: Runemu) -> list[Launcher]:
+    """The kickstarts link of the package, and the ROM each default config names."""
+    package = Script(AMIBERRY_PACKAGE, read(AMIBERRY_PACKAGE))
+    links = [link for link in package.links(BIOS) if link.bios_is_target]
+    if len(links) != 1 or links[0].bios_side.rstrip("/") != BIOS:
+        raise package.missing(f"ln -s {BIOS} to the kickstarts folder")
+    link = links[0]
+    installed = re.sub(r"^\$\{INSTALL\}", "", link.link)
+    path, table_line = runemu.script("AMIBERRY")
+    script = Script(path, read(path))
+    launchers = [Launcher(
+        emulator="AMIBERRY", build="standalone", kind="links", names=[],
+        case_insensitive=False, search_dir=BIOS,
+        passes_to=f"{installed} is a symlink to it", fails_when_missing=False,
+        source_ref=source_ref([(AMIBERRY_PACKAGE, link.lines)]),
+    )]
+    for number, line in script.code():
+        copied = re.search(r"cp\s+\$AMIBERRY_CONFIG_DIR/(\S+\.uae)\s", line)
+        if copied is None:
+            continue
+        config = f"{AMIBERRY_CONFIGS}/{copied.group(1)}"
+        found = [
+            s for s in settings(read(config))
+            if s.key == "kickstart_rom_file" and s.value.startswith("$(FILE_PATH)/")
+        ]
+        rom_path = [s for s in settings(read(config)) if s.key == "amiberry.rom_path"]
+        if len(found) != 1 or len(rom_path) != 1:
+            raise LauncherPatternMissing(config, "kickstart_rom_file under $(FILE_PATH)")
+        if rom_path[0].value.rstrip("/").rsplit("/", 1)[-1] != installed.rstrip("/").rsplit("/", 1)[-1]:
+            raise LauncherPatternMissing(config, f"amiberry.rom_path naming {installed}")
+        condition_line = max(
+            n for n, text in script.code()
+            if n < number and re.match(r"\s*(if|else)\b", text)
+        )
+        branch = script.lines[condition_line - 1].strip()
+        if branch == "else":
+            opened = max(
+                n for n, text in script.code()
+                if n < condition_line and re.match(r"\s*if\b", text)
+            )
+            test = _shell_test(script, opened)
+            condition = f"not ({test})"
+        else:
+            test = _shell_test(script, condition_line)
+            condition = test
+        launchers.append(Launcher(
+            emulator="AMIBERRY", build="standalone", kind="fixed_name",
+            slot=copied.group(1).removesuffix("-default.uae").removeprefix("Amiga"),
+            names=[found[0].value.removeprefix("$(FILE_PATH)/")], case_insensitive=False,
+            search_dir=BIOS,
+            passes_to=f"{copied.group(1)} kickstart_rom_file, $(FILE_PATH) = {rom_path[0].value}",
+            fails_when_missing=False,
+            depends_on_option=condition,
+            source_ref=source_ref([
+                (RUNEMU, [table_line]), (path, [condition_line, number]),
+                (config, [rom_path[0].line, found[0].line]),
+            ]),
+        ))
+    if len(launchers) == 1:
+        raise script.missing("cp of a default .uae config")
+    return launchers
+
+
+def _shell_test(script: Script, line: int) -> str:
+    """The test of `if <test>; then` at a line."""
+    found = re.match(r"\s*if\s+(.*?);\s*then\s*$", script.lines[line - 1])
+    if found is None:
+        raise script.missing(f"if ...; then at line {line}")
+    return found.group(1)
+
+
+def scummvm(read: Read) -> list[Launcher]:
+    """The paths scummvm.sh writes into the libretro core's scummvm.ini."""
+    path = SCRIPTS["SCUMMVMSA"]
+    script = Script(path, read(path))
+    branch, _ = script.find(r'^\s*"libretro"\)', "libretro branch")
+    launchers = []
+    for number, line in script.code():
+        if number < branch:
+            continue
+        written = re.search(r"s/\^/(\w*path)=((?:\\/[^\\/n]+)+)\\n/", line)
+        if written is None:
+            continue
+        folder = written.group(2).replace("\\/", "/")
+        if "/roms/bios/" not in folder + "/" or written.group(1) == "browser_lastpath":
+            continue
+        launchers.append(Launcher(
+            emulator="scummvm", build="libretro", kind="passes_path", names=[],
+            search_dir=folder, passes_to=f"scummvm.ini {written.group(1)}",
+            fails_when_missing=False, depends_on_option="scummvm.sh libretro",
+            source_ref=source_ref([(path, [branch, number])]),
+        ))
+    if not launchers:
+        raise script.missing("a scummvm.ini path written under /roms/bios")
+    return launchers
+
+
+def freej2me(read: Read) -> list[Launcher]:
+    """`if [ ! -f <bios>/<jar> ]; then cp <shipped jar> <bios>`."""
+    launchers = []
+    for core, path in FREEJ2ME:
+        script = Script(path, read(path))
+        tested, test = script.find(
+            rf'if \[ ! -f "{re.escape(BIOS)}/([^"]+)" \]', "jar presence test"
+        )
+        jar = test.group(1)
+        lines = [tested]
+        source = None
+        for number, line in script.code():
+            copied = re.match(rf"\s*cp\s+(?:-\S+\s+)*(\S+/{re.escape(jar)})\s+{re.escape(BIOS)}/?\s*$", line)
+            if copied:
+                lines.append(number)
+                source = copied.group(1)
+        if source is None:
+            raise script.missing(f"cp of {jar} into {BIOS}")
+        launchers.append(Launcher(
+            emulator=core, build="libretro", kind="copies", names=[jar],
+            case_insensitive=False, search_dir=posixpath.dirname(source), passes_to=BIOS,
+            fails_when_missing=False, source_ref=source_ref([(path, lines)]),
+        ))
+    return launchers
+
+
+def easyrpg(read: Read) -> list[Launcher]:
+    """Each install_bios call: a runtime package downloaded and unpacked under BIOS."""
+    script = Script(EASYRPG, read(EASYRPG))
+    url_line, url = script.find(r'curl -Lo "\$\{BIOS\}" "([^"]+)"', "curl of the package")
+    launchers = []
+    for number, line in script.code():
+        call = re.match(r"\s*install_bios\s+(\S+)\s+(\S+)\s*$", line)
+        if call is None:
+            continue
+        used: set[int] = {number, url_line}
+        source = script.expand(url.group(1), number, used, {"BIOS": call.group(1)})
+        install = script.value("INSTALLPATH", number)
+        if install is None or install.value.rstrip("/") != BIOS:
+            raise script.missing(f"INSTALLPATH {BIOS}")
+        used.add(install.line)
+        launchers.append(Launcher(
+            emulator="easyrpg", build="libretro", kind="copies", names=[],
+            search_dir=source, passes_to=f"{BIOS}/{call.group(2)}",
+            fails_when_missing=False, source_ref=source_ref([(EASYRPG, used)]),
+        ))
+    if not launchers:
+        raise script.missing("install_bios call")
+    return launchers
+
+
+def extract_launchers(read: Read) -> list[dict[str, Any]]:
+    """Every launcher record, read through `read(repo-relative path)`."""
+    runemu = Runemu(read)
+    launchers = [
+        *retroarch(read), *retrorun(read), *mame(runemu),
+        *atari800(read), *scummvm(read), *freej2me(read), *easyrpg(read),
+        *amiberry(read, runemu), *hatari(read, runemu), *pico8(read, runemu),
+        *piemu(read, runemu), *yabasanshiro(read, runemu),
+    ]
+    return [launcher.entry() for launcher in launchers]

scripts/scraper/amberelec_scraper.py

@@ -23,6 +23,9 @@ Verification (batocera-systems at that commit):
   (packages/amberelec/profile.d/99-distribution.conf:320) passes
   `--strictfilter`, which the script's argument switch (lines 665-687) does
   not handle: that call prints nothing and exits 0.
+
+The launcher layer is read at the same commit (amberelec_launchers): runemu.py,
+the scripts its standalone table runs and the configs their packages install.
 """
 
 from __future__ import annotations
@@ -32,6 +35,7 @@ import re
 import urllib.error
 import urllib.request
 
+from .amberelec_launchers import extract_launchers
 from .base_scraper import BaseScraper, BiosRequirement, github_headers
 from .batocera_scraper import (
     SYSTEM_SLUG_MAP as BATOCERA_SLUGS,
@@ -155,6 +159,9 @@ class Scraper(BaseScraper):
             "standalone_cores": standalone,
             "dual_mode_cores": dual,
             "systems": systems,
+            "launchers": extract_launchers(
+                lambda path: self._fetch_text(_RAW.format(ref=self.commit) + path)
+            ),
         }
 
 

scripts/scraper/batocera_launchers.py

@@ -58,6 +58,29 @@ def _config_option(node: ast.AST) -> tuple[str, Any] | None:
     return node.args[0].value, default
 
 
+def _option_names(node: ast.AST) -> list[str]:
+    """Options a node reads: system.config.get*(o), system.config[o], system.isOptSet(o)."""
+    found: list[str] = []
+    for child in ast.walk(node):
+        option = _config_option(child)
+        if option is None and (
+            isinstance(child, ast.Subscript)
+            and ast.unparse(child.value) == "system.config"
+            and isinstance(child.slice, ast.Constant)
+        ):
+            option = (str(child.slice.value), None)
+        if option is None and (
+            isinstance(child, ast.Call)
+            and ast.unparse(child.func) == "system.isOptSet"
+            and child.args
+            and isinstance(child.args[0], ast.Constant)
+        ):
+            option = (str(child.args[0].value), None)
+        if option is not None and option[0] not in found:
+            found.append(option[0])
+    return found
+
+
 def path_constants(source: Source) -> dict[str, str]:
     """batoceraPaths' module constants that are plain paths."""
     constants: dict[str, str] = {}
@@ -92,12 +115,18 @@ def _path_value(node: ast.AST | None, constants: dict[str, str]) -> str | None:
     return None
 
 
-def generator_names(source: Source) -> dict[str, list[str]]:
-    """Generator module -> the emulator or core names configgen loads it for."""
+def generator_names(source: Source, legacy_map: bool = True) -> dict[str, list[str]]:
+    """Generator module -> the emulator or core names configgen loads it for.
+
+    legacy_map is false for a configgen written before _LEGACY_GENERATOR_MAP
+    (KNULLI's), whose importer has the plain map only.
+    """
     names: dict[str, list[str]] = {}
     plain = source.literal(source.assignment("_GENERATOR_MAP").value, "_GENERATOR_MAP")
     for emulator, (module, _cls) in plain.items():
         names.setdefault(module, []).append(emulator)
+    if not legacy_map:
+        return names
     legacy = source.literal(
         source.assignment("_LEGACY_GENERATOR_MAP").value, "_LEGACY_GENERATOR_MAP"
     )
@@ -108,14 +137,23 @@ def generator_names(source: Source) -> dict[str, list[str]]:
 
 
 class Configgen:
-    """The configgen tree at one revision, read through `read(path)`."""
+    """The configgen tree at one revision, read through `read(path)`.
 
-    def __init__(self, read: Callable[[str], str]) -> None:
+    root is the configgen package in the repository: Batocera's by default,
+    KNULLI keeps the same package under its own path.
+    """
+
+    def __init__(
+        self, read: Callable[[str], str], root: str = CONFIGGEN, legacy_map: bool = True
+    ) -> None:
         self._read = read
         self._sources: dict[str, Source] = {}
-        self.paths = path_constants(self.source(PATHS_MODULE))
+        self.generators = f"{root}/generators"
+        self.importer = f"{self.generators}/importer.py"
+        self.libretro_options = f"{self.generators}/libretro/libretroOptions.py"
+        self.paths = path_constants(self.source(f"{root}/batoceraPaths.py"))
         self.bios = self.paths["BIOS"]
-        self._generators = generator_names(self.source(IMPORTER))
+        self._generators = generator_names(self.source(self.importer), legacy_map)
 
     def source(self, path: str) -> Source:
         if path not in self._sources:
@@ -123,7 +161,7 @@ class Configgen:
         return self._sources[path]
 
     def generator(self, module: str) -> Source:
-        return self.source(f"{GENERATORS}/{module.replace('.', '/')}.py")
+        return self.source(f"{self.generators}/{module.replace('.', '/')}.py")
 
     def helper(self, module: str, helper: str) -> Source:
         """A module the generator imports with `from . import helper`."""
@@ -137,7 +175,7 @@ class Configgen:
         )
         if not imported:
             generator.fail(f"does not import {helper}")
-        return self.source(f"{GENERATORS}/{module.split('.')[0]}/{helper}.py")
+        return self.source(f"{self.generators}/{module.split('.')[0]}/{helper}.py")
 
     def emulator(self, module: str) -> str:
         """The name configgen loads `module` for (importer.get_generator)."""
@@ -146,7 +184,7 @@ class Configgen:
         package, _, leaf = module.partition(".")
         if leaf == f"{package}Generator":
             return package
-        raise LauncherShapeError(f"{IMPORTER}: no emulator loads {module}")
+        raise LauncherShapeError(f"{self.importer}: no emulator loads {module}")
 
     def bios_name(self, node: ast.AST) -> str | None:
         """The name under /userdata/bios that an expression spells, if any."""
@@ -386,15 +424,28 @@ def _loaded(stmt: ast.stmt) -> set[str]:
     return {n.id for n in ast.walk(value) if isinstance(n, ast.Name)} if value else set()
 
 
+def _branch_assigns(stmt: ast.stmt, needed: set[str]) -> bool:
+    """Whether an if statement assigns one of the names in any branch."""
+    return isinstance(stmt, ast.If) and any(
+        _assigned_name(node) in needed for node in ast.walk(stmt)
+    )
+
+
 def _trace(body: list[ast.stmt], index: int) -> list[ast.stmt]:
     """body[index], the assignments before it that bind what it reads, and
-    the command-line additions that read any of those names."""
+    the command-line additions that read any of those names.
+
+    An if statement that assigns a name read is kept with what its test
+    reads (KNULLI's `if system.isOptSet("model") ...: machine = ...`)."""
     kept = [body[index]]
     needed = _loaded(body[index])
     for stmt in reversed(body[:index]):
         if _assigned_name(stmt) in needed:
             kept.append(stmt)
             needed |= _loaded(stmt)
+        elif _branch_assigns(stmt, needed):
+            kept.append(stmt)
+            needed |= {n.id for n in ast.walk(stmt) if isinstance(n, ast.Name)}
     produced = {name for stmt in kept if (name := _assigned_name(stmt)) is not None}
     kept += [
         stmt for stmt in body
@@ -435,15 +486,23 @@ def hatari(cg: Configgen) -> list[Launcher]:
                 and isinstance(parts[1], ast.FormattedValue)
             ):
                 marker, prefix = node.test.left.value, parts[0].value
-    tos_path = src.assignment("tos_path", finder).value
-    if isinstance(tos_path, ast.BinOp) and isinstance(tos_path.right, ast.JoinedStr):
-        parts = tos_path.right.values
-        if (
-            len(parts) == 3
-            and all(isinstance(p, ast.FormattedValue) for p in parts[:2])
-            and isinstance(parts[2], ast.Constant)
-        ):
-            suffix = parts[2].value
+    # Batocera: tos_path = biosdir / f"{biosversion}{v_language}.img".
+    # KNULLI: filename = f"{biosversion}{v_language}.img".
+    names_built = []
+    for node in ast.walk(finder):
+        value = node.value if isinstance(node, ast.Assign) else None
+        if isinstance(value, ast.BinOp) and isinstance(value.op, ast.Div):
+            value = value.right
+        if isinstance(value, ast.JoinedStr):
+            parts = value.values
+            if (
+                len(parts) == 3
+                and all(isinstance(p, ast.FormattedValue) for p in parts[:2])
+                and isinstance(parts[2], ast.Constant)
+            ):
+                names_built.append(parts[2].value)
+    if len(names_built) == 1:
+        suffix = names_built[0]
     if marker is None or prefix is None or suffix is None:
         src.fail("findBestTos no longer builds <tos{version} or etos*><language>.img")
     raised = [node for node in ast.walk(finder) if isinstance(node, ast.Raise)]
@@ -466,10 +525,7 @@ def hatari(cg: Configgen) -> list[Launcher]:
         src.fail("findBestTos is not given a batoceraPaths directory")
     search_dir = cg.paths[call.args[0].id]
     kept = _trace(body, index)
-    options = [
-        option[0] for stmt in kept for node in ast.walk(stmt)
-        if (option := _config_option(node)) is not None
-    ]
+    options = list(dict.fromkeys(name for stmt in kept for name in _option_names(stmt)))
     spans = [src.span(stmt) for stmt in kept] + [(finder.lineno, finder.end_lineno)]
     args: list[str] = []
     second = call.args[1] if len(call.args) > 1 else None
@@ -484,6 +540,14 @@ def hatari(cg: Configgen) -> list[Launcher]:
                     args.append("{machine}")
                 elif isinstance(element, ast.Name) and element.id == tos_var:
                     args.append("<file>")
+                elif (
+                    isinstance(element, ast.BinOp)
+                    and isinstance(element.right, ast.Name)
+                    and element.right.id == tos_var
+                    and _path_value(element.left, cg.paths) == search_dir
+                ):
+                    # KNULLI returns the bare name and joins it: BIOS / tos.
+                    args.append("<file>")
     if "<file>" not in args:
         src.fail("the TOS findBestTos returns is not put on the command line")
 
@@ -563,7 +627,7 @@ def melonds(cg: Configgen) -> list[Launcher]:
         node for node in ast.walk(generate)
         if isinstance(node, ast.Assign)
         and ast.unparse(node.targets[0]).startswith("base_config['Emu']['ConsoleType']")
-        and _config_option(node.value) is not None
+        and _option_names(node.value)
     ]
     enabled = [
         (key, value) for key, value in zip(base.value.keys, base.value.values)
@@ -574,7 +638,7 @@ def melonds(cg: Configgen) -> list[Launcher]:
     ]
     if len(console) != 1 or len(enabled) != 1:
         src.fail("generate no longer sets Emu.ConsoleType from an option with external BIOS on")
-    option = _config_option(console[0].value)[0]  # type: ignore[index]
+    option = _option_names(console[0].value)[0]  # type: ignore[attr-defined]
     launchers = []
     for key, value in zip(base.value.keys, base.value.values):
         if not isinstance(value, ast.Dict) or not isinstance(key, ast.Constant):
@@ -805,7 +869,7 @@ def environment_directories(cg: Configgen) -> list[Launcher]:
     for module, paths_module in ENVIRONMENT_GENERATORS:
         src = cg.generator(module)
         constants = dict(cg.paths)
-        for node in cg.source(f"{GENERATORS}/{paths_module}").tree.body:
+        for node in cg.source(f"{cg.generators}/{paths_module}").tree.body:
             name = _assigned_name(node)
             value = _path_value(getattr(node, "value", None), constants)
             if name is not None and value is not None:
@@ -840,7 +904,7 @@ FILE_OPTIONS = (("_o2em_options", "o2em_bios"), ("_neocd_options", "neocd_bios")
 
 def core_file_options(cg: Configgen) -> list[Launcher]:
     """_set_from_system(settings, option, system, [name], default=...)."""
-    src = cg.source(LIBRETRO_OPTIONS)
+    src = cg.source(cg.libretro_options)
     table = src.assignment("_option_functions").value
     if not isinstance(table, ast.Dict):
         src.fail("_option_functions is not a dict display")

scripts/scraper/knulli_scraper.py

@@ -19,9 +19,10 @@ Verification (knulli-systems at the release tag):
   10834d00, es-app/src/ApiSystem.cpp:764-774) and by knulli-support:69.
 
 es_systems.yml and configgen-defaults.yml have Batocera's shape, so the cores
-are read with Batocera's code. The configgen generators do not: the hatari,
-melonds and gsplus generators and libretroOptions.py are older forms than the
-ones batocera_launchers reads, so no launcher layer is written.
+are read with Batocera's code. The launcher layer comes from KNULLI's own
+configgen package and board scripts at the same tag (knulli_launchers):
+Batocera's extractors where a generator is still Batocera's code, KNULLI's
+where it is older or its own.
 """
 
 from __future__ import annotations
@@ -44,6 +45,7 @@ from .batocera_scraper import (
     requirements_from_systems,
 )
 from .es_emulators import attach_system_cores
+from .knulli_launchers import extract_launchers
 
 PLATFORM_NAME = "knulli"
 
@@ -124,6 +126,9 @@ class Scraper(BaseScraper):
         # configgen hands a libretro core /userdata/saves/<system>, as
         # Batocera's does: the systems each core runs for are its save folders.
         result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)
+        result["launchers"] = extract_launchers(
+            lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path)
+        )
         return result
 
 

scripts/scraper/retrobat_launchers.py

@@ -320,6 +320,16 @@ def feature_choices(
     return values, src.lines(feature.start(), close + len("</feature>"))
 
 
+# A copy record reads `names` from `search_dir` and hands them to the path in
+# `passes_to`. When the copy is skipped is a condition of the record.
+TARGET_ABSENT = "the target does not exist"
+
+
+def _joined(*parts: str | None) -> str | None:
+    """Conditions of a record, joined, None when there are none."""
+    return ", ".join(part for part in parts if part) or None
+
+
 def _record(
     emulator: str, build: str, kind: str, names: list[str], ref: str, **fields: object
 ) -> Record:
@@ -611,9 +621,9 @@ def hatari_libretro(sources: dict[str, Source]) -> list[Record]:
             source_ref((src.path, [head, src.lines(shared.start(), copy.end())])),
             case_insensitive=True,
             search_dir="/".join(["bios", *folders]),
-            passes_to=f"copied to bios/{own.group(2)}/<value>, only when the target does not exist",
+            passes_to=f"bios/{own.group(2)}/<value>",
             fails_when_missing=False,
-            depends_on_option=key,
+            depends_on_option=_joined(key, TARGET_ABSENT),
         )
     )
     return records
@@ -661,10 +671,9 @@ def mesen(sources: dict[str, Source]) -> list[Record]:
                 source_ref((src.path, [src.statement_span(block)])),
                 case_insensitive=True,
                 search_dir="bios",
-                passes_to="copied to <mesen>/Firmware/"
-                + ", ".join(destinations)
-                + ", only when the target does not exist",
+                passes_to=", ".join(f"<mesen>/Firmware/{d}" for d in destinations),
                 fails_when_missing=fails,
+                depends_on_option=TARGET_ABSENT,
             )
         )
     return records
@@ -804,11 +813,9 @@ def winuae(sources: dict[str, Source]) -> list[Record]:
             ),
             case_insensitive=True,
             search_dir=search,
-            passes_to=(
-                f"copied to <WHDBoot>/{'/'.join(_strings(folder.group(1)))}/<name>"
-                " from the system's Kickstart candidates"
-            ),
+            passes_to=f"<WHDBoot>/{'/'.join(_strings(folder.group(1)))}/<name>",
             fails_when_missing=False,
+            depends_on_option="the system's Kickstart candidates",
         )
     )
     return records
@@ -2480,10 +2487,12 @@ def dolphin(sources: dict[str, Source]) -> list[Record]:
             ),
             case_insensitive=True,
             search_dir="bios",
-            passes_to=f"first found copied to {destination[0] if destination else target.group(1)},"
-            " skipped when the target is as large and not older",
+            passes_to=destination[0] if destination else target.group(1),
             fails_when_missing=f"warning '{_unescape(warning.group(1))}' when the target is absent too",
-            depends_on_option=", ".join(conditions) or None,
+            depends_on_option=_joined(
+                *conditions, "first name found",
+                "the target is absent, smaller or older",
+            ),
         )
     )
     return _covered(src, records, skip=(r"DolphinTool\.exe",))
@@ -2578,9 +2587,9 @@ def _copies_if_absent(
                 _path_parts(src, *returns, *[s for c in copies for s in c[3]]),
                 case_insensitive=True,
                 search_dir=folders.pop(),
-                passes_to=f"copied to {destination}/<name>, only when the target does not exist",
+                passes_to=f"{destination}/<name>",
                 fails_when_missing=False,
-                depends_on_option=", ".join(conditions) or None,
+                depends_on_option=_joined(*conditions, TARGET_ABSENT),
             )
         )
     if not records:
@@ -2619,7 +2628,9 @@ def capsimg(sources: dict[str, Source]) -> list[Record]:
     record["source_ref"] = source_ref(
         (src.path, [(src.line(cores.start()), core_lines[1]), src.statement_span(block)])
     )
-    record["depends_on_option"] = f'core in {cores.group(1)}, rom extension "{gate.group(1)}"'
+    record["depends_on_option"] = _joined(
+        f'core in {cores.group(1)}, rom extension "{gate.group(1)}"', TARGET_ABSENT
+    )
     return [record]
 
 
@@ -3108,9 +3119,9 @@ def xm6pro(sources: dict[str, Source]) -> list[Record]:
         _path_parts(src, (src.line(table.start()), lines[1]), src.lines(method.start(), body[1])),
         case_insensitive=True,
         search_dir=folder[0],
-        passes_to="copied to <emulator>/<name>, only when the target does not exist",
+        passes_to="<emulator>/<name>",
         fails_when_missing=False,
-        depends_on_option=", ".join(conditions) or None,
+        depends_on_option=_joined(*conditions, TARGET_ABSENT),
     )
     return _covered(src, [record])