From ffad06487cde8f136dfb2534f00776cd0fddce5b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:47:23 +0100 Subject: [PATCH] feat: read the amberelec launcher layer amberelec_launchers reads runemu.py with ast and, at the commit the scraper pins, the scripts its standalone table runs and the configs their packages install: RetroArch and retrorun directories, the MAME -rp path, atari800.cfg, Hatari's default TOS, amiberry's kickstarts link and defaults, yabasanshiro, PICO-8, piemu, scummvm, freej2me and easyrpg. 17 records. The freej2me-lr.jar and pico8_64 notes now name the file in a sentence about the emulator, as test_launcher_aliases asks once a launcher records it. --- emulators/freej2me.yml | 2 +- emulators/lexaloffle.yml | 2 +- platforms/amberelec.yml | 181 +++++++++++ scripts/scraper/amberelec_launchers.py | 429 +++++++++++++++++++++++++ scripts/scraper/amberelec_scraper.py | 7 + tests/test_launchers_amberelec.py | 345 ++++++++++++++++++++ 6 files changed, 964 insertions(+), 2 deletions(-) create mode 100644 scripts/scraper/amberelec_launchers.py create mode 100644 tests/test_launchers_amberelec.py diff --git a/emulators/freej2me.yml b/emulators/freej2me.yml index 45a5bc303..85c6af334 100644 --- a/emulators/freej2me.yml +++ b/emulators/freej2me.yml @@ -31,4 +31,4 @@ files: required: true installed_on: [retrobat, rocknix] source_ref: "src/libretro/freej2me_libretro.c:487-490 (GET_SYSTEM_DIRECTORY + fill_pathname_join + java -jar)" - note: "Not a BIOS dump. Java emulator loaded by the native core at startup via execvp('java', '-jar', path). Built from source via 'ant'. Must be in the system directory root. ROCKNIX builds the core and the jar from freej2me-plus 1.52, which keeps the freej2me-lr.jar name, and copies the jar into its BIOS folder before each launch when it is missing." + note: "Not a BIOS dump. Java emulator the native core joins as freej2me-lr.jar to the system directory and starts via execvp('java', '-jar', path). Built from source via 'ant'. Must be in the system directory root. ROCKNIX builds the core and the jar from freej2me-plus 1.52, which keeps the freej2me-lr.jar name, and copies the jar into its BIOS folder before each launch when it is missing." diff --git a/emulators/lexaloffle.yml b/emulators/lexaloffle.yml index 1fa22616a..28f058674 100644 --- a/emulators/lexaloffle.yml +++ b/emulators/lexaloffle.yml @@ -130,7 +130,7 @@ files: md5: 70ab37eab4e7d162d36d9b2d91b84867 sha1: 4ed23b6578a2d93d67aa856791dc70e95d68a3f4 description: "PICO-8 0.2.7 aarch64 executable, system libSDL2" - note: "Only in pico-8_0.2.7_raspi.zip, the build for 64-bit Raspberry Pi OS, linked against the system SDL2. AmberELEC and ROCKNIX run it from roms/pico-8/, not from bios/. AmberELEC stops with a message when it is missing. ROCKNIX picks it when HW_ARCH is aarch64. Recalbox renames bios/pico-8/pico8_64 to pico8 before the first run, and a missing file fails the rename." + note: "pico8_64 is only in pico-8_0.2.7_raspi.zip, the build for 64-bit Raspberry Pi OS, linked against the system SDL2. AmberELEC and ROCKNIX run it from roms/pico-8/, not from bios/. AmberELEC stops with a message when it is missing. ROCKNIX picks it when HW_ARCH is aarch64. Recalbox renames bios/pico-8/pico8_64 to pico8 before the first run, and a missing file fails the rename." source_ref: "pico-8 0.2.7 raspi readme_raspi.txt:3-11, amberelec packages/amberelec/sources/pico-8.sh:51-54, amberelec packages/amberelec/sources/scripts/runemu.py:123, rocknix projects/ROCKNIX/packages/emulators/standalone/pico-8/scripts/start_pico8.sh:8-17, 35-40, 46-47, recalbox projects/configgen/configgen/generators/pico8/pico8Generator.py:14-19, dArkOS pico8/pico8.sh:47-60, 128-135" - name: pico8_dyn diff --git a/platforms/amberelec.yml b/platforms/amberelec.yml index c6e472540..a4085d839 100644 --- a/platforms/amberelec.yml +++ b/platforms/amberelec.yml @@ -3172,3 +3172,184 @@ systems: cores: - fuse - ep128emu_core +launchers: +- emulator: retroarch + build: libretro + kind: passes_path + names: [] + search_dir: ~/roms/bios + passes_to: retroarch.cfg system_directory + fails_when_missing: false + source_ref: "packages/games/tools/retroarch/sources/retroarch.cfg:611" +- emulator: retrorun + build: libretro + kind: passes_path + names: [] + search_dir: /roms/bios + passes_to: command line -d + fails_when_missing: false + source_ref: "packages/games/tools/retrorun/retrorun.sh:583" +- emulator: mame + build: libretro + kind: passes_path + names: [] + search_dir: /roms/bios/mame;/roms/bios;/roms/arcade;/roms/mame + passes_to: .cmd -rp + fails_when_missing: false + depends_on_option: self.emulator == 'mame' + source_ref: "packages/amberelec/sources/scripts/runemu.py:233, 252, 255" +- emulator: atari800 + build: libretro + kind: fixed_name + names: + - ATARIOSA.ROM + - ATARIXL.ROM + - 5200.rom + - 5200a.rom + - BASIC_Rev_A.rom + - BASIC_Rev_B.rom + - ATARIBAS.ROM + - ATARIOSB.ROM + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: atari800.cfg ROM_OS_A_PAL, ROM_OS_BB01R2, ROM_5200, ROM_5200_A, ROM_BASIC_A, + ROM_BASIC_B, ROM_BASIC_C, ROM_400/800_CUSTOM + fails_when_missing: false + source_ref: "packages/games/libretro/atari800/patches/atari800-01-add-emuelec-platform.patch:27,\ + \ packages/amberelec/config/distribution/configs/atari800.cfg:3, 8, 16-22" +- emulator: scummvm + build: libretro + kind: passes_path + names: [] + search_dir: /roms/bios/scummvm/theme + passes_to: scummvm.ini themepath + fails_when_missing: false + depends_on_option: scummvm.sh libretro + source_ref: "packages/games/emulators/scummvmsa/scummvm.sh:34, 42" +- emulator: scummvm + build: libretro + kind: passes_path + names: [] + search_dir: /roms/bios/scummvm/extra + passes_to: scummvm.ini extrapath + fails_when_missing: false + depends_on_option: scummvm.sh libretro + source_ref: "packages/games/emulators/scummvmsa/scummvm.sh:34, 43" +- emulator: freej2me + build: libretro + kind: copies + names: + - freej2me-lr.jar + case_insensitive: false + search_dir: /usr/config/distribution/freej2me + passes_to: /storage/roms/bios + fails_when_missing: false + source_ref: "packages/games/libretro/freej2me/freej2me.sh:11-12, 44" +- emulator: freej2me-plus + build: libretro + kind: copies + names: + - freej2me-plus-lr.jar + case_insensitive: false + search_dir: /usr/config/distribution/freej2me + passes_to: /storage/roms/bios + fails_when_missing: false + source_ref: "packages/games/libretro/freej2me-plus/freej2me-plus.sh:11-12, 44" +- emulator: easyrpg + build: libretro + kind: copies + names: [] + search_dir: "https://dl.degica.com/rpgmakerweb/run-time-packages/rpg2000_rtp_installer.exe" + passes_to: /storage/roms/bios/rtp/2000 + fails_when_missing: false + source_ref: "packages/games/libretro/easyrpg/easyrpg.sh:23, 42, 46, 60" +- emulator: easyrpg + build: libretro + kind: copies + names: [] + search_dir: "https://dl.degica.com/rpgmakerweb/run-time-packages/rpg2003_rtp_installer.zip" + passes_to: /storage/roms/bios/rtp/2003 + fails_when_missing: false + source_ref: "packages/games/libretro/easyrpg/easyrpg.sh:23, 42, 46, 61" +- emulator: AMIBERRY + build: standalone + kind: links + names: [] + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: /usr/config/amiberry/kickstarts is a symlink to it + fails_when_missing: false + source_ref: "packages/games/emulators/amiberry/package.mk:35" +- emulator: AMIBERRY + build: standalone + kind: fixed_name + slot: A1200 + names: + - kick40068.A1200.rom + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: "AmigaA1200-default.uae kickstart_rom_file, $(FILE_PATH) = /storage/.config/amiberry/kickstarts/" + fails_when_missing: false + depends_on_option: "[ `echo $1 | egrep 'AGA|CD32' | wc -l` -eq 1 -o `echo \"$AMIBERRY_TMP_DIR\"\ + /* | egrep 'AGA|CD32' | wc -l` -eq 1 ]" + source_ref: "packages/amberelec/sources/scripts/runemu.py:111, packages/games/emulators/amiberry/scripts/amiberry.sh:57-59,\ + \ packages/games/emulators/amiberry/config/conf/AmigaA1200-default.uae:35, 85" +- emulator: AMIBERRY + build: standalone + kind: fixed_name + slot: A500 + names: + - kick34005.A500.rom + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: "AmigaA500-default.uae kickstart_rom_file, $(FILE_PATH) = /storage/.config/amiberry/kickstarts/" + fails_when_missing: false + depends_on_option: "not ([ `echo $1 | egrep 'AGA|CD32' | wc -l` -eq 1 -o `echo \"\ + $AMIBERRY_TMP_DIR\"/* | egrep 'AGA|CD32' | wc -l` -eq 1 ])" + source_ref: "packages/amberelec/sources/scripts/runemu.py:111, packages/games/emulators/amiberry/scripts/amiberry.sh:60-62,\ + \ packages/games/emulators/amiberry/config/conf/AmigaA500-default.uae:35, 85" +- emulator: HATARISA + build: standalone + kind: fixed_name + names: + - tos.img + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: "Atari-ST-default.cfg [ROM] szTosImageFileName" + fails_when_missing: false + source_ref: "packages/amberelec/sources/scripts/runemu.py:113, packages/games/emulators/hatarisa/scripts/hatari.sh:36,\ + \ 66, packages/games/emulators/hatarisa/config/conf/Atari-ST-default.cfg:200" +- emulator: pico8 + build: standalone + kind: fixed_name + names: + - pico8_64 + - pico8.dat + case_insensitive: false + search_dir: /storage/roms/pico-8 + passes_to: exec + fails_when_missing: Missing Pico-8 binaries! + source_ref: "packages/amberelec/sources/scripts/runemu.py:123, packages/amberelec/sources/pico-8.sh:51-54" +- emulator: piemu + build: standalone + kind: fixed_name + names: + - all.bin + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: mkpfi argument + fails_when_missing: false + source_ref: "packages/amberelec/sources/scripts/runemu.py:124, packages/games/emulators/piemu/bin/piemu.sh:8-10,\ + \ 33, 37, 50" +- emulator: yabasanshiroSA + build: standalone + kind: fixed_name + names: + - saturn_bios.bin + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: command line -b + fails_when_missing: false + depends_on_option: use_hlebios + source_ref: "packages/amberelec/sources/scripts/runemu.py:127, packages/games/emulators/yabasanshiroSA/yabasanshiro.sh:36,\ + \ 63-65, 78" diff --git a/scripts/scraper/amberelec_launchers.py b/scripts/scraper/amberelec_launchers.py new file mode 100644 index 000000000..725c8cced --- /dev/null +++ b/scripts/scraper/amberelec_launchers.py @@ -0,0 +1,429 @@ +"""AmberELEC's launcher layer, read from runemu.py and its emulator scripts. + +EmulationStation hands every launch to runemu.py. A libretro core runs in +RetroArch or retrorun with the system directory their configs name, MAME +machines get a .cmd file with a -rp search path, and every other emulator +runs the script its standalone_emulators table names (/usr/bin/.sh, +installed from the package). Those scripts, and the configs their packages +install, say which file under /storage/roms/bios is opened: yabasanshiro.sh +tests saturn_bios.bin and passes it with -b, Hatari's default config names +tos.img, the atari800 core is built to read a config naming its OS ROMs. + +runemu.py is read with ast, the scripts with shell_launchers' readers. An +extractor whose construct is gone raises instead of returning nothing. +""" + +from __future__ import annotations + +import ast +import posixpath +import re +from collections.abc import Callable +from typing import Any + +try: + from .configgen_launchers import Launcher, Source + from .shell_launchers import ( + LauncherPatternMissing, + Script, + describe_keys, + settings, + source_ref, + ) +except ImportError: + from configgen_launchers import Launcher, Source # type: ignore[no-redef] + from shell_launchers import ( # type: ignore[no-redef] + LauncherPatternMissing, + Script, + describe_keys, + settings, + source_ref, + ) + +BIOS = "/storage/roms/bios" +RUNEMU = "packages/amberelec/sources/scripts/runemu.py" +RETROARCH_CFG = "packages/games/tools/retroarch/sources/retroarch.cfg" +RETRORUN = "packages/games/tools/retrorun/retrorun.sh" +HATARI_CONFIG = "packages/games/emulators/hatarisa/config/conf/Atari-ST-default.cfg" +ATARI800_CONFIG = "packages/amberelec/config/distribution/configs/atari800.cfg" +ATARI800_PATCH = "packages/games/libretro/atari800/patches/atari800-01-add-emuelec-platform.patch" +AMIBERRY_PACKAGE = "packages/games/emulators/amiberry/package.mk" +AMIBERRY_CONFIGS = "packages/games/emulators/amiberry/config/conf" +FREEJ2ME = ( + ("freej2me", "packages/games/libretro/freej2me/freej2me.sh"), + ("freej2me-plus", "packages/games/libretro/freej2me-plus/freej2me-plus.sh"), +) +EASYRPG = "packages/games/libretro/easyrpg/easyrpg.sh" + +# The script each standalone_emulators entry runs, as the package installs it +# to /usr/bin. scripts() checks each against runemu's table. +SCRIPTS = { + "AMIBERRY": "packages/games/emulators/amiberry/scripts/amiberry.sh", + "HATARISA": "packages/games/emulators/hatarisa/scripts/hatari.sh", + "SCUMMVMSA": "packages/games/emulators/scummvmsa/scummvm.sh", + "pico8": "packages/amberelec/sources/pico-8.sh", + "piemu": "packages/games/emulators/piemu/bin/piemu.sh", + "yabasanshiroSA": "packages/games/emulators/yabasanshiroSA/yabasanshiro.sh", +} + +Read = Callable[[str], str] + + +class Runemu: + """runemu.py, parsed: the standalone table and the RetroArch command.""" + + def __init__(self, read: Read) -> None: + self.src = Source(RUNEMU, read(RUNEMU)) + node = self.src.assignment("standalone_emulators") + if not isinstance(node.value, ast.Dict): + self.src.fail("standalone_emulators is not a dict display") + self.table: dict[str, tuple[str, int]] = {} + for key, value in zip(node.value.keys, node.value.values): + if not ( + isinstance(key, ast.Constant) + and isinstance(value, ast.Call) + and len(value.args) >= 2 + and isinstance(value.args[1], ast.List) + ): + self.src.fail(f"entry at line {getattr(key, 'lineno', '?')} is not StandaloneEmulator(name, [args])") + args = [a.value for a in value.args[1].elts if isinstance(a, ast.Constant)] + script = next((a for a in args if str(a).endswith(".sh")), None) + if script is None: + self.src.fail(f"{ast.unparse(key)} runs no .sh script") + self.table[str(key.value)] = (str(script), value.lineno) + + def script(self, emulator: str) -> tuple[str, int]: + """The repository script runemu runs for emulator, and the table line.""" + if emulator not in self.table: + self.src.fail(f"standalone_emulators has no {emulator}") + installed, line = self.table[emulator] + path = SCRIPTS[emulator] + if posixpath.basename(installed) != posixpath.basename(path): + self.src.fail(f"{emulator} runs {installed}, not {posixpath.basename(path)}") + return path, line + + +def retroarch(read: Read) -> list[Launcher]: + """system_directory of the retroarch.cfg the retroarch package installs.""" + found = [s for s in settings(read(RETROARCH_CFG)) if s.key == "system_directory"] + if len(found) != 1: + raise LauncherPatternMissing(RETROARCH_CFG, "one system_directory") + return [Launcher( + emulator="retroarch", build="libretro", kind="passes_path", names=[], + search_dir=found[0].value, passes_to="retroarch.cfg system_directory", + fails_when_missing=False, source_ref=source_ref([(RETROARCH_CFG, [found[0].line])]), + )] + + +def retrorun(read: Read) -> list[Launcher]: + """The -d directory retrorun.sh passes to retrorun.""" + script = Script(RETRORUN, read(RETRORUN)) + line, found = script.find(r"^\s*/usr/bin/retrorun\s.*\s-d\s+(\S+)", "retrorun -d") + return [Launcher( + emulator="retrorun", build="libretro", kind="passes_path", names=[], + search_dir=found.group(1).strip('"'), passes_to="command line -d", + fails_when_missing=False, source_ref=source_ref([(RETRORUN, [line])]), + )] + + +def mame(runemu: Runemu) -> list[Launcher]: + """The -rp search path of the .cmd file runemu writes for a MAME machine.""" + src = runemu.src + commands = [ + node for node in ast.walk(src.function("get_retroarch_command")) + if isinstance(node, ast.JoinedStr) + and any(isinstance(p, ast.Constant) and " -rp " in str(p.value) for p in node.values) + ] + if not commands: + src.fail("get_retroarch_command writes no -rp search path") + paths = [] + for node in commands: + text = "".join(str(p.value) for p in node.values if isinstance(p, ast.Constant)) + paths.append(text.split(" -rp ", 1)[1].split()[0]) + if len(set(paths)) != 1: + src.fail(f"the MAME commands differ in their search path: {sorted(set(paths))}") + guard = next( + node for node in ast.walk(src.function("get_retroarch_command")) + if isinstance(node, ast.If) and ast.unparse(node.test) == "self.emulator == 'mame'" + ) + return [Launcher( + emulator="mame", build="libretro", kind="passes_path", names=[], + search_dir=paths[0], passes_to=".cmd -rp", fails_when_missing=False, + depends_on_option=ast.unparse(guard.test), + source_ref=src.ref([ + src.span(guard.test), *((n.lineno, n.end_lineno or n.lineno) for n in commands), + ]), + )] + + +def _bios_files(path: str, text: str) -> tuple[list[tuple[tuple[str, str], str]], set[int]]: + """((section, key), name under BIOS) for each config key whose value is a BIOS file.""" + prefix = BIOS + "/" + found = [s for s in settings(text) if s.value.startswith(prefix)] + if not found: + raise LauncherPatternMissing(path, f"key naming a file under {BIOS}") + return [((s.section, s.key), s.value[len(prefix):]) for s in found], {s.line for s in found} + + +def hatari(read: Read, runemu: Runemu) -> list[Launcher]: + """szTosImageFileName of the default config hatari.sh copies and passes.""" + path, table_line = runemu.script("HATARISA") + script = Script(path, read(path)) + name = posixpath.basename(HATARI_CONFIG) + copied, _ = script.find(rf"^\s*cp\s+\S*/{re.escape(name)}\s", f"cp of {name}") + passed, _ = script.find(r"\shatari\s+--configfile\s", "hatari --configfile") + files, lines = _bios_files(HATARI_CONFIG, read(HATARI_CONFIG)) + return [Launcher( + emulator="HATARISA", build="standalone", kind="fixed_name", + names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS, + passes_to=describe_keys(name, tuple(k for k, _n in files)), + fails_when_missing=False, + source_ref=source_ref([ + (RUNEMU, [table_line]), (path, [copied, passed]), (HATARI_CONFIG, lines), + ]), + )] + + +def atari800(read: Read) -> list[Launcher]: + """The ROM keys of the config the atari800 core is built to read.""" + patch = Script(ATARI800_PATCH, read(ATARI800_PATCH)) + line, found = patch.find(r'DEFAULT_CFG_NAME="\\"([^"\\]+)\\""', "DEFAULT_CFG_NAME") + built = found.group(1) + if not built.endswith("/" + ATARI800_CONFIG.split("/config/", 1)[1]): + raise LauncherPatternMissing(ATARI800_PATCH, f"DEFAULT_CFG_NAME naming {ATARI800_CONFIG}") + files, lines = _bios_files(ATARI800_CONFIG, read(ATARI800_CONFIG)) + return [Launcher( + emulator="atari800", build="libretro", kind="fixed_name", + names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS, + passes_to=describe_keys(posixpath.basename(ATARI800_CONFIG), tuple(k for k, _n in files)), + fails_when_missing=False, + source_ref=source_ref([(ATARI800_PATCH, [line]), (ATARI800_CONFIG, lines)]), + )] + + +def yabasanshiro(read: Read, runemu: Runemu) -> list[Launcher]: + """`if [ -e ]; then BIOS="-b "`, skipped when HLE is chosen.""" + path, table_line = runemu.script("yabasanshiroSA") + script = Script(path, read(path)) + tested, test = script.find(rf'if \[ -e "({re.escape(BIOS)}/[^"]+)" \]', "BIOS presence test") + flag_line, flag = script.find( + rf'BIOS="(-\w+) {re.escape(test.group(1))}"', "BIOS flag", tested + ) + hle_line, hle = script.find(r'get_setting "(\w*hle\w*)"', "HLE setting") + guard_line, _ = script.find(r'if \[ "\$\{HLEBIOS\}" != 1 \]', "HLE test") + used_line, _ = script.find(r"\$\{BIOS\}", "BIOS on the command line", flag_line + 1) + return [Launcher( + emulator="yabasanshiroSA", build="standalone", kind="fixed_name", + names=[test.group(1)[len(BIOS) + 1:]], case_insensitive=False, search_dir=BIOS, + passes_to=f"command line {flag.group(1)}", fails_when_missing=False, + depends_on_option=hle.group(1), + source_ref=source_ref([ + (RUNEMU, [table_line]), + (path, [hle_line, guard_line, tested, flag_line, used_line]), + ]), + )] + + +def pico8(read: Read, runemu: Runemu) -> list[Launcher]: + """`if [ ! -f A ] || [ ! -f B ]; then text_viewer ...; else A ...`.""" + path, table_line = runemu.script("pico8") + script = Script(path, read(path)) + line, test = script.find(r'if \[ ! -f "([^"]+)" \] \|\| \[ ! -f "([^"]+)" \]', "binary test") + message_line, message = script.find(r'text_viewer .*-t "([^"]+)"', "missing message", line) + binary = test.group(1) + run_line, _ = script.find(rf"^\s*{re.escape(binary)}\s", f"{binary} run", message_line) + folders = {posixpath.dirname(test.group(1)), posixpath.dirname(test.group(2))} + if len(folders) != 1: + raise script.missing("both binaries in one folder") + return [Launcher( + emulator="pico8", build="standalone", kind="fixed_name", + names=[posixpath.basename(test.group(1)), posixpath.basename(test.group(2))], + case_insensitive=False, search_dir=folders.pop(), passes_to="exec", + fails_when_missing=message.group(1), + source_ref=source_ref([(RUNEMU, [table_line]), (path, [line, message_line, run_line])]), + )] + + +def piemu(read: Read, runemu: Runemu) -> list[Launcher]: + """The BIOS file piemu.sh hands to mkpfi to build the flash image.""" + path, table_line = runemu.script("piemu") + script = Script(path, read(path)) + names: list[str] = [] + lines: set[int] = set() + for number, line in script.code(): + found = re.search(r'/usr/bin/mkpfi\s+"([^"]+)"', line) + if found is None: + continue + used: set[int] = {number} + value = script.expand(found.group(1), number, used) + if value.startswith(BIOS + "/") and value[len(BIOS) + 1:] not in names: + names.append(value[len(BIOS) + 1:]) + lines |= used + if not names: + raise script.missing(f"mkpfi of a file under {BIOS}") + return [Launcher( + emulator="piemu", build="standalone", kind="fixed_name", names=names, + case_insensitive=False, search_dir=BIOS, passes_to="mkpfi argument", + fails_when_missing=False, + source_ref=source_ref([(RUNEMU, [table_line]), (path, lines)]), + )] + + +def amiberry(read: Read, runemu: Runemu) -> list[Launcher]: + """The kickstarts link of the package, and the ROM each default config names.""" + package = Script(AMIBERRY_PACKAGE, read(AMIBERRY_PACKAGE)) + links = [link for link in package.links(BIOS) if link.bios_is_target] + if len(links) != 1 or links[0].bios_side.rstrip("/") != BIOS: + raise package.missing(f"ln -s {BIOS} to the kickstarts folder") + link = links[0] + installed = re.sub(r"^\$\{INSTALL\}", "", link.link) + path, table_line = runemu.script("AMIBERRY") + script = Script(path, read(path)) + launchers = [Launcher( + emulator="AMIBERRY", build="standalone", kind="links", names=[], + case_insensitive=False, search_dir=BIOS, + passes_to=f"{installed} is a symlink to it", fails_when_missing=False, + source_ref=source_ref([(AMIBERRY_PACKAGE, link.lines)]), + )] + for number, line in script.code(): + copied = re.search(r"cp\s+\$AMIBERRY_CONFIG_DIR/(\S+\.uae)\s", line) + if copied is None: + continue + config = f"{AMIBERRY_CONFIGS}/{copied.group(1)}" + found = [ + s for s in settings(read(config)) + if s.key == "kickstart_rom_file" and s.value.startswith("$(FILE_PATH)/") + ] + rom_path = [s for s in settings(read(config)) if s.key == "amiberry.rom_path"] + if len(found) != 1 or len(rom_path) != 1: + raise LauncherPatternMissing(config, "kickstart_rom_file under $(FILE_PATH)") + if rom_path[0].value.rstrip("/").rsplit("/", 1)[-1] != installed.rstrip("/").rsplit("/", 1)[-1]: + raise LauncherPatternMissing(config, f"amiberry.rom_path naming {installed}") + condition_line = max( + n for n, text in script.code() + if n < number and re.match(r"\s*(if|else)\b", text) + ) + branch = script.lines[condition_line - 1].strip() + if branch == "else": + opened = max( + n for n, text in script.code() + if n < condition_line and re.match(r"\s*if\b", text) + ) + test = _shell_test(script, opened) + condition = f"not ({test})" + else: + test = _shell_test(script, condition_line) + condition = test + launchers.append(Launcher( + emulator="AMIBERRY", build="standalone", kind="fixed_name", + slot=copied.group(1).removesuffix("-default.uae").removeprefix("Amiga"), + names=[found[0].value.removeprefix("$(FILE_PATH)/")], case_insensitive=False, + search_dir=BIOS, + passes_to=f"{copied.group(1)} kickstart_rom_file, $(FILE_PATH) = {rom_path[0].value}", + fails_when_missing=False, + depends_on_option=condition, + source_ref=source_ref([ + (RUNEMU, [table_line]), (path, [condition_line, number]), + (config, [rom_path[0].line, found[0].line]), + ]), + )) + if len(launchers) == 1: + raise script.missing("cp of a default .uae config") + return launchers + + +def _shell_test(script: Script, line: int) -> str: + """The test of `if ; then` at a line.""" + found = re.match(r"\s*if\s+(.*?);\s*then\s*$", script.lines[line - 1]) + if found is None: + raise script.missing(f"if ...; then at line {line}") + return found.group(1) + + +def scummvm(read: Read) -> list[Launcher]: + """The paths scummvm.sh writes into the libretro core's scummvm.ini.""" + path = SCRIPTS["SCUMMVMSA"] + script = Script(path, read(path)) + branch, _ = script.find(r'^\s*"libretro"\)', "libretro branch") + launchers = [] + for number, line in script.code(): + if number < branch: + continue + written = re.search(r"s/\^/(\w*path)=((?:\\/[^\\/n]+)+)\\n/", line) + if written is None: + continue + folder = written.group(2).replace("\\/", "/") + if "/roms/bios/" not in folder + "/" or written.group(1) == "browser_lastpath": + continue + launchers.append(Launcher( + emulator="scummvm", build="libretro", kind="passes_path", names=[], + search_dir=folder, passes_to=f"scummvm.ini {written.group(1)}", + fails_when_missing=False, depends_on_option="scummvm.sh libretro", + source_ref=source_ref([(path, [branch, number])]), + )) + if not launchers: + raise script.missing("a scummvm.ini path written under /roms/bios") + return launchers + + +def freej2me(read: Read) -> list[Launcher]: + """`if [ ! -f / ]; then cp `.""" + launchers = [] + for core, path in FREEJ2ME: + script = Script(path, read(path)) + tested, test = script.find( + rf'if \[ ! -f "{re.escape(BIOS)}/([^"]+)" \]', "jar presence test" + ) + jar = test.group(1) + lines = [tested] + source = None + for number, line in script.code(): + copied = re.match(rf"\s*cp\s+(?:-\S+\s+)*(\S+/{re.escape(jar)})\s+{re.escape(BIOS)}/?\s*$", line) + if copied: + lines.append(number) + source = copied.group(1) + if source is None: + raise script.missing(f"cp of {jar} into {BIOS}") + launchers.append(Launcher( + emulator=core, build="libretro", kind="copies", names=[jar], + case_insensitive=False, search_dir=posixpath.dirname(source), passes_to=BIOS, + fails_when_missing=False, source_ref=source_ref([(path, lines)]), + )) + return launchers + + +def easyrpg(read: Read) -> list[Launcher]: + """Each install_bios call: a runtime package downloaded and unpacked under BIOS.""" + script = Script(EASYRPG, read(EASYRPG)) + url_line, url = script.find(r'curl -Lo "\$\{BIOS\}" "([^"]+)"', "curl of the package") + launchers = [] + for number, line in script.code(): + call = re.match(r"\s*install_bios\s+(\S+)\s+(\S+)\s*$", line) + if call is None: + continue + used: set[int] = {number, url_line} + source = script.expand(url.group(1), number, used, {"BIOS": call.group(1)}) + install = script.value("INSTALLPATH", number) + if install is None or install.value.rstrip("/") != BIOS: + raise script.missing(f"INSTALLPATH {BIOS}") + used.add(install.line) + launchers.append(Launcher( + emulator="easyrpg", build="libretro", kind="copies", names=[], + search_dir=source, passes_to=f"{BIOS}/{call.group(2)}", + fails_when_missing=False, source_ref=source_ref([(EASYRPG, used)]), + )) + if not launchers: + raise script.missing("install_bios call") + return launchers + + +def extract_launchers(read: Read) -> list[dict[str, Any]]: + """Every launcher record, read through `read(repo-relative path)`.""" + runemu = Runemu(read) + launchers = [ + *retroarch(read), *retrorun(read), *mame(runemu), + *atari800(read), *scummvm(read), *freej2me(read), *easyrpg(read), + *amiberry(read, runemu), *hatari(read, runemu), *pico8(read, runemu), + *piemu(read, runemu), *yabasanshiro(read, runemu), + ] + return [launcher.entry() for launcher in launchers] diff --git a/scripts/scraper/amberelec_scraper.py b/scripts/scraper/amberelec_scraper.py index 0f729fae8..1ce5dfbe2 100644 --- a/scripts/scraper/amberelec_scraper.py +++ b/scripts/scraper/amberelec_scraper.py @@ -23,6 +23,9 @@ Verification (batocera-systems at that commit): (packages/amberelec/profile.d/99-distribution.conf:320) passes `--strictfilter`, which the script's argument switch (lines 665-687) does not handle: that call prints nothing and exits 0. + +The launcher layer is read at the same commit (amberelec_launchers): runemu.py, +the scripts its standalone table runs and the configs their packages install. """ from __future__ import annotations @@ -32,6 +35,7 @@ import re import urllib.error import urllib.request +from .amberelec_launchers import extract_launchers from .base_scraper import BaseScraper, BiosRequirement, github_headers from .batocera_scraper import ( SYSTEM_SLUG_MAP as BATOCERA_SLUGS, @@ -155,6 +159,9 @@ class Scraper(BaseScraper): "standalone_cores": standalone, "dual_mode_cores": dual, "systems": systems, + "launchers": extract_launchers( + lambda path: self._fetch_text(_RAW.format(ref=self.commit) + path) + ), } diff --git a/tests/test_launchers_amberelec.py b/tests/test_launchers_amberelec.py new file mode 100644 index 000000000..c0ba27d36 --- /dev/null +++ b/tests/test_launchers_amberelec.py @@ -0,0 +1,345 @@ +"""AmberELEC's launcher layer, read from runemu.py and its emulator scripts. + +The fixtures are cut-down copies of what each extractor reads at AmberELEC +a21975bf (prerelease-20261019_1835). Records must name the files in the +order the code writes them, cite the fixture's own lines, and a construct +that is gone must stop the scrape. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import sys +import textwrap +import unittest +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from scraper import amberelec_launchers as al # noqa: E402 +from scraper.configgen_launchers import LauncherShapeError # noqa: E402 + + +def text(block: str) -> str: + return textwrap.dedent(block).lstrip("\n") + + +RUNEMU = text( + """ + standalone_emulators = { + 'AMIBERRY': StandaloneEmulator('amiberry', ['/usr/bin/amiberry.sh', '']), + 'HATARISA': StandaloneEmulator('hatari', ['/usr/bin/hatari.sh', '']), + 'pico8': StandaloneEmulator('pico8_64', ['/usr/bin/pico-8.sh', '']), + 'piemu': StandaloneEmulator('piemu', ['/usr/bin/bash', '-l', '/usr/bin/piemu.sh', '']), + 'yabasanshiroSA': StandaloneEmulator('yabasanshiro', ['/usr/bin/yabasanshiro.sh', '']), + } + + class EmuRunner(): + def get_retroarch_command(self, shader_arg): + if self.emulator == 'mame': + if rom_path: + cmd = f'{mame_system} -{mame_slot} "{rom_path}" -rp /roms/bios/mame;/roms/bios -cfg_directory /x' + else: + cmd = f'{mame_system} -rp /roms/bios/mame;/roms/bios -cfg_directory /x' + """ +) + +YABASANSHIRO = text( + """ + #!/bin/bash + get_setting "use_hlebios" + HLEBIOS='0' + BIOS="" + if [ "${HLEBIOS}" != 1 ]; then + if [ -e "/storage/roms/bios/saturn_bios.bin" ]; then + BIOS="-b /storage/roms/bios/saturn_bios.bin" + else + text_viewer -e -w -t "Notice! Yabasanshiro BIOS Missing!" -m "not found" + fi + fi + { yabasanshiro -r 2 -i "${1}" ${BIOS}; } > /tmp/logs/exec.log 2>&1 + """ +) + +PICO8 = text( + """ + #!/bin/bash + mkdir -p "/storage/roms/pico-8" + if [ ! -f "/storage/roms/pico-8/pico8_64" ] || [ ! -f "/storage/roms/pico-8/pico8.dat" ]; then + text_viewer -e -w -t "Missing Pico-8 binaries!" -m "Extract your purchased pico8_64." + else + /storage/roms/pico-8/pico8_64 -home -root_path /storage/roms/pico-8 ${OPTIONS} "${CART}" + fi + """ +) + +PIEMU = text( + """ + #!/bin/bash + ROMS_DIR="/storage/roms" + BIOS_DIR="$ROMS_DIR/bios" + if [[ "$EXTENSION" == "pfs" ]]; then + /usr/bin/mkpfi "$BIOS_DIR/all.bin" + fi + """ +) + +HATARI_SH = text( + """ + #!/bin/bash + HATARI_CONFIG_DIR=/storage/.config/hatari/conf + cp $HATARI_CONFIG_DIR/Atari-ST-default.cfg "$HATARI_TMP_CONFIG" + nice -n -19 hatari --configfile "$HATARI_TMP_CONFIG" --disk-a "$1" >> $HATARI_LOG 2>&1 + """ +) + +HATARI_CFG = text( + """ + [ROM] + szTosImageFileName = /storage/roms/bios/tos.img + bPatchTos = TRUE + """ +) + +ATARI800_CFG = text( + """ + Atari 800 Emulator, Version 3.1.0 + ROM_OS_A_NTSC= + ROM_OS_A_PAL=/storage/roms/bios/ATARIOSA.ROM + ROM_5200=/storage/roms/bios/5200.rom + """ +) + +ATARI800_PATCH = text( + """ + + LDFLAGS += -static-libgcc + + CFLAGS += -DDEFAULT_CFG_NAME="\\"/storage/.config/distribution/configs/atari800.cfg\\"" + """ +) + +AMIBERRY_MK = text( + """ + makeinstall_target() { + mkdir -p ${INSTALL}/usr/config/amiberry + ln -s /storage/roms/bios ${INSTALL}/usr/config/amiberry/kickstarts + } + """ +) + +AMIBERRY_SH = text( + """ + #!/bin/bash + AMIBERRY_CONFIG_DIR=$AMIBERRY_DIR/conf + AMIBERRY_SET_CONF() { + if [ `echo $1 | egrep 'AGA|CD32' | wc -l` -eq 1 ]; then + cp $AMIBERRY_CONFIG_DIR/AmigaA1200-default.uae "$AMIBERRY_TMP_CONFIG" + else + cp $AMIBERRY_CONFIG_DIR/AmigaA500-default.uae "$AMIBERRY_TMP_CONFIG" + fi + } + """ +) + + +def uae(rom: str) -> str: + return f"amiberry.rom_path=/storage/.config/amiberry/kickstarts/\nkickstart_rom_file=$(FILE_PATH)/{rom}\n" + + +SCUMMVM_SH = text( + """ + #!/bin/bash + case $1 in + "libretro") + sed -i '2,2s/^/themepath=\\/roms\\/bios\\/scummvm\\/theme\\n/' ${BIOSPATH}/scummvm.ini + sed -i '2,2s/^/extrapath=\\/roms\\/bios\\/scummvm\\/extra\\n/' ${BIOSPATH}/scummvm.ini + sed -i '2,2s/^/browser_lastpath=\\/roms\\/bios\\/scummvm\\/extra\\n/' ${BIOSPATH}/scummvm.ini + ;; + esac + """ +) + +FREEJ2ME = text( + """ + #!/bin/bash + if [ ! -f "/storage/roms/bios/{jar}" ]; then + cp /usr/config/distribution/freej2me/{jar} /storage/roms/bios + fi + """ +) + +EASYRPG = text( + """ + #!/bin/bash + install_bios() { + BIOS="$1" + curl -Lo "${BIOS}" "${PKG_URL}/${BIOS}" + } + INSTALLPATH="/storage/roms/bios" + PKG_URL="https://dl.degica.com/rpgmakerweb/run-time-packages" + install_bios rpg2000_rtp_installer.exe rtp/2000 + """ +) + + +def tree(**overrides: str) -> dict[str, str]: + files = { + al.RUNEMU: RUNEMU, + al.RETROARCH_CFG: 'savefile_directory = "~/saves"\nsystem_directory = "~/roms/bios"\n', + al.RETRORUN: '/usr/bin/retrorun -c "${RRCONF}" -s /storage/roms/"$3" -d /roms/bios "$EMU" "$2"\n', + al.SCRIPTS["yabasanshiroSA"]: YABASANSHIRO, + al.SCRIPTS["pico8"]: PICO8, + al.SCRIPTS["piemu"]: PIEMU, + al.SCRIPTS["HATARISA"]: HATARI_SH, + al.HATARI_CONFIG: HATARI_CFG, + al.ATARI800_CONFIG: ATARI800_CFG, + al.ATARI800_PATCH: ATARI800_PATCH, + al.AMIBERRY_PACKAGE: AMIBERRY_MK, + al.SCRIPTS["AMIBERRY"]: AMIBERRY_SH, + f"{al.AMIBERRY_CONFIGS}/AmigaA1200-default.uae": uae("kick40068.A1200.rom"), + f"{al.AMIBERRY_CONFIGS}/AmigaA500-default.uae": uae("kick34005.A500.rom"), + al.SCRIPTS["SCUMMVMSA"]: SCUMMVM_SH, + al.EASYRPG: EASYRPG, + **{path: FREEJ2ME.replace("{jar}", f"{core}-lr.jar") for core, path in al.FREEJ2ME}, + } + files.update(overrides) + return files + + +def read(**overrides: str): + return tree(**overrides).__getitem__ + + +class Frontends(unittest.TestCase): + def test_retroarch_system_directory(self): + (record,) = al.retroarch(read()) + self.assertEqual(record.search_dir, "~/roms/bios") + self.assertEqual(record.source_ref, f"{al.RETROARCH_CFG}:2") + + def test_retrorun_directory(self): + (record,) = al.retrorun(read()) + self.assertEqual((record.search_dir, record.passes_to), ("/roms/bios", "command line -d")) + + def test_mame_search_path_of_both_commands(self): + (record,) = al.mame(al.Runemu(read())) + self.assertEqual(record.search_dir, "/roms/bios/mame;/roms/bios") + self.assertEqual(record.depends_on_option, "self.emulator == 'mame'") + self.assertEqual(record.source_ref, f"{al.RUNEMU}:11, 13, 15") + + def test_a_retroarch_config_without_the_key_raises(self): + with self.assertRaises(LauncherShapeError): + al.retroarch(read(**{al.RETROARCH_CFG: "x = 1\n"})) + + +class Standalone(unittest.TestCase): + def test_yabasanshiro_flag_after_the_presence_test(self): + (record,) = al.yabasanshiro(read(), al.Runemu(read())) + self.assertEqual(record.names, ["saturn_bios.bin"]) + self.assertEqual(record.passes_to, "command line -b") + self.assertIs(record.fails_when_missing, False) + self.assertEqual(record.depends_on_option, "use_hlebios") + self.assertEqual( + record.source_ref, + f"{al.RUNEMU}:6, {al.SCRIPTS['yabasanshiroSA']}:2, 5-7, 12", + ) + + def test_pico8_binaries_outside_bios(self): + (record,) = al.pico8(read(), al.Runemu(read())) + self.assertEqual(record.names, ["pico8_64", "pico8.dat"]) + self.assertEqual(record.search_dir, "/storage/roms/pico-8") + self.assertEqual(record.fails_when_missing, "Missing Pico-8 binaries!") + self.assertEqual(record.source_ref, f"{al.RUNEMU}:4, {al.SCRIPTS['pico8']}:3-6") + + def test_piemu_flash_image_source(self): + (record,) = al.piemu(read(), al.Runemu(read())) + self.assertEqual(record.names, ["all.bin"]) + self.assertEqual(record.source_ref, f"{al.RUNEMU}:5, {al.SCRIPTS['piemu']}:2-5") + + def test_hatari_default_config(self): + (record,) = al.hatari(read(), al.Runemu(read())) + self.assertEqual(record.names, ["tos.img"]) + self.assertEqual(record.passes_to, "Atari-ST-default.cfg [ROM] szTosImageFileName") + self.assertEqual( + record.source_ref, + f"{al.RUNEMU}:3, {al.SCRIPTS['HATARISA']}:3-4, {al.HATARI_CONFIG}:2", + ) + + def test_amiberry_link_and_one_kickstart_per_config(self): + link, a1200, a500 = al.amiberry(read(), al.Runemu(read())) + self.assertEqual(link.kind, "links") + self.assertEqual(link.passes_to, "/usr/config/amiberry/kickstarts is a symlink to it") + self.assertEqual((a1200.slot, a1200.names), ("A1200", ["kick40068.A1200.rom"])) + self.assertEqual((a500.slot, a500.names), ("A500", ["kick34005.A500.rom"])) + self.assertEqual(a500.depends_on_option, f"not ({a1200.depends_on_option})") + + def test_a_table_running_another_script_raises(self): + broken = RUNEMU.replace("/usr/bin/yabasanshiro.sh", "/usr/bin/yaba.sh") + with self.assertRaises(LauncherShapeError): + al.yabasanshiro(read(), al.Runemu(read(**{al.RUNEMU: broken}))) + + def test_a_bios_test_written_another_way_raises(self): + broken = YABASANSHIRO.replace('if [ -e "/storage', 'if test -e "/storage') + with self.assertRaises(LauncherShapeError): + al.yabasanshiro(read(**{al.SCRIPTS["yabasanshiroSA"]: broken}), al.Runemu(read())) + + +class Cores(unittest.TestCase): + def test_atari800_built_to_read_the_shipped_config(self): + (record,) = al.atari800(read()) + self.assertEqual(record.names, ["ATARIOSA.ROM", "5200.rom"]) + self.assertEqual(record.source_ref, f"{al.ATARI800_PATCH}:2, {al.ATARI800_CONFIG}:3-4") + + def test_atari800_built_for_another_config_raises(self): + broken = ATARI800_PATCH.replace("atari800.cfg", "other.cfg") + with self.assertRaises(LauncherShapeError): + al.atari800(read(**{al.ATARI800_PATCH: broken})) + + def test_scummvm_paths_written_under_bios(self): + theme, extra = al.scummvm(read()) + self.assertEqual(theme.search_dir, "/roms/bios/scummvm/theme") + self.assertEqual(extra.passes_to, "scummvm.ini extrapath") + + def test_freej2me_copies_the_shipped_jar(self): + plain, plus = al.freej2me(read()) + self.assertEqual((plain.kind, plain.names), ("copies", ["freej2me-lr.jar"])) + self.assertEqual(plus.names, ["freej2me-plus-lr.jar"]) + self.assertEqual(plain.search_dir, "/usr/config/distribution/freej2me") + + def test_easyrpg_download_per_runtime(self): + (record,) = al.easyrpg(read()) + self.assertEqual( + record.search_dir, + "https://dl.degica.com/rpgmakerweb/run-time-packages/rpg2000_rtp_installer.exe", + ) + self.assertEqual(record.passes_to, "/storage/roms/bios/rtp/2000") + + def test_every_extractor_runs_on_the_fixture(self): + records = al.extract_launchers(read()) + # One easyrpg runtime in the fixture, two at the pinned commit. + self.assertEqual(len(records), 16) + + +class PlatformFile(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.config = yaml.safe_load((REPO_ROOT / "platforms" / "amberelec.yml").read_text()) + + def test_the_layer_is_written(self): + self.assertTrue(self.config["launchers"]) + + def test_every_reference_is_in_the_repository(self): + for record in self.config["launchers"]: + self.assertTrue(record["source_ref"].startswith("packages/"), record) + + def test_every_emulator_is_one_amberelec_names(self): + cores = {*self.config["cores"], *self.config.get("standalone_cores", [])} + named = {n for record in self.config["launchers"] for n in record["emulator"].split(", ")} + # RetroArch and retrorun themselves, the two runners of every core. + self.assertEqual(sorted(named - cores), ["retroarch", "retrorun"]) + + +if __name__ == "__main__": + unittest.main()