From 10e4d925d096133504e8c9ed37d099dea0e5ba49 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:47:09 +0100 Subject: [PATCH 1/7] refactor: root the configgen reader at a package Configgen takes the configgen package path and whether the importer has a legacy map, so KNULLI's copy at package/system/knulli-configgen reads with the same extractors. hatari accepts the bare filename KNULLI builds and joins to BIOS, and options read by subscript or isOptSet. melonds accepts a console type read by subscript. Batocera's records are unchanged at batocera-43.1. --- scripts/scraper/batocera_launchers.py | 118 ++++++++++++++++++++------ 1 file changed, 91 insertions(+), 27 deletions(-) diff --git a/scripts/scraper/batocera_launchers.py b/scripts/scraper/batocera_launchers.py index ea6fc69aa..67f03887e 100644 --- a/scripts/scraper/batocera_launchers.py +++ b/scripts/scraper/batocera_launchers.py @@ -58,6 +58,29 @@ def _config_option(node: ast.AST) -> tuple[str, Any] | None: return node.args[0].value, default +def _option_names(node: ast.AST) -> list[str]: + """Options a node reads: system.config.get*(o), system.config[o], system.isOptSet(o).""" + found: list[str] = [] + for child in ast.walk(node): + option = _config_option(child) + if option is None and ( + isinstance(child, ast.Subscript) + and ast.unparse(child.value) == "system.config" + and isinstance(child.slice, ast.Constant) + ): + option = (str(child.slice.value), None) + if option is None and ( + isinstance(child, ast.Call) + and ast.unparse(child.func) == "system.isOptSet" + and child.args + and isinstance(child.args[0], ast.Constant) + ): + option = (str(child.args[0].value), None) + if option is not None and option[0] not in found: + found.append(option[0]) + return found + + def path_constants(source: Source) -> dict[str, str]: """batoceraPaths' module constants that are plain paths.""" constants: dict[str, str] = {} @@ -92,12 +115,18 @@ def _path_value(node: ast.AST | None, constants: dict[str, str]) -> str | None: return None -def generator_names(source: Source) -> dict[str, list[str]]: - """Generator module -> the emulator or core names configgen loads it for.""" +def generator_names(source: Source, legacy_map: bool = True) -> dict[str, list[str]]: + """Generator module -> the emulator or core names configgen loads it for. + + legacy_map is false for a configgen written before _LEGACY_GENERATOR_MAP + (KNULLI's), whose importer has the plain map only. + """ names: dict[str, list[str]] = {} plain = source.literal(source.assignment("_GENERATOR_MAP").value, "_GENERATOR_MAP") for emulator, (module, _cls) in plain.items(): names.setdefault(module, []).append(emulator) + if not legacy_map: + return names legacy = source.literal( source.assignment("_LEGACY_GENERATOR_MAP").value, "_LEGACY_GENERATOR_MAP" ) @@ -108,14 +137,23 @@ def generator_names(source: Source) -> dict[str, list[str]]: class Configgen: - """The configgen tree at one revision, read through `read(path)`.""" + """The configgen tree at one revision, read through `read(path)`. - def __init__(self, read: Callable[[str], str]) -> None: + root is the configgen package in the repository: Batocera's by default, + KNULLI keeps the same package under its own path. + """ + + def __init__( + self, read: Callable[[str], str], root: str = CONFIGGEN, legacy_map: bool = True + ) -> None: self._read = read self._sources: dict[str, Source] = {} - self.paths = path_constants(self.source(PATHS_MODULE)) + self.generators = f"{root}/generators" + self.importer = f"{self.generators}/importer.py" + self.libretro_options = f"{self.generators}/libretro/libretroOptions.py" + self.paths = path_constants(self.source(f"{root}/batoceraPaths.py")) self.bios = self.paths["BIOS"] - self._generators = generator_names(self.source(IMPORTER)) + self._generators = generator_names(self.source(self.importer), legacy_map) def source(self, path: str) -> Source: if path not in self._sources: @@ -123,7 +161,7 @@ class Configgen: return self._sources[path] def generator(self, module: str) -> Source: - return self.source(f"{GENERATORS}/{module.replace('.', '/')}.py") + return self.source(f"{self.generators}/{module.replace('.', '/')}.py") def helper(self, module: str, helper: str) -> Source: """A module the generator imports with `from . import helper`.""" @@ -137,7 +175,7 @@ class Configgen: ) if not imported: generator.fail(f"does not import {helper}") - return self.source(f"{GENERATORS}/{module.split('.')[0]}/{helper}.py") + return self.source(f"{self.generators}/{module.split('.')[0]}/{helper}.py") def emulator(self, module: str) -> str: """The name configgen loads `module` for (importer.get_generator).""" @@ -146,7 +184,7 @@ class Configgen: package, _, leaf = module.partition(".") if leaf == f"{package}Generator": return package - raise LauncherShapeError(f"{IMPORTER}: no emulator loads {module}") + raise LauncherShapeError(f"{self.importer}: no emulator loads {module}") def bios_name(self, node: ast.AST) -> str | None: """The name under /userdata/bios that an expression spells, if any.""" @@ -386,15 +424,28 @@ def _loaded(stmt: ast.stmt) -> set[str]: return {n.id for n in ast.walk(value) if isinstance(n, ast.Name)} if value else set() +def _branch_assigns(stmt: ast.stmt, needed: set[str]) -> bool: + """Whether an if statement assigns one of the names in any branch.""" + return isinstance(stmt, ast.If) and any( + _assigned_name(node) in needed for node in ast.walk(stmt) + ) + + def _trace(body: list[ast.stmt], index: int) -> list[ast.stmt]: """body[index], the assignments before it that bind what it reads, and - the command-line additions that read any of those names.""" + the command-line additions that read any of those names. + + An if statement that assigns a name read is kept with what its test + reads (KNULLI's `if system.isOptSet("model") ...: machine = ...`).""" kept = [body[index]] needed = _loaded(body[index]) for stmt in reversed(body[:index]): if _assigned_name(stmt) in needed: kept.append(stmt) needed |= _loaded(stmt) + elif _branch_assigns(stmt, needed): + kept.append(stmt) + needed |= {n.id for n in ast.walk(stmt) if isinstance(n, ast.Name)} produced = {name for stmt in kept if (name := _assigned_name(stmt)) is not None} kept += [ stmt for stmt in body @@ -435,15 +486,23 @@ def hatari(cg: Configgen) -> list[Launcher]: and isinstance(parts[1], ast.FormattedValue) ): marker, prefix = node.test.left.value, parts[0].value - tos_path = src.assignment("tos_path", finder).value - if isinstance(tos_path, ast.BinOp) and isinstance(tos_path.right, ast.JoinedStr): - parts = tos_path.right.values - if ( - len(parts) == 3 - and all(isinstance(p, ast.FormattedValue) for p in parts[:2]) - and isinstance(parts[2], ast.Constant) - ): - suffix = parts[2].value + # Batocera: tos_path = biosdir / f"{biosversion}{v_language}.img". + # KNULLI: filename = f"{biosversion}{v_language}.img". + names_built = [] + for node in ast.walk(finder): + value = node.value if isinstance(node, ast.Assign) else None + if isinstance(value, ast.BinOp) and isinstance(value.op, ast.Div): + value = value.right + if isinstance(value, ast.JoinedStr): + parts = value.values + if ( + len(parts) == 3 + and all(isinstance(p, ast.FormattedValue) for p in parts[:2]) + and isinstance(parts[2], ast.Constant) + ): + names_built.append(parts[2].value) + if len(names_built) == 1: + suffix = names_built[0] if marker is None or prefix is None or suffix is None: src.fail("findBestTos no longer builds .img") raised = [node for node in ast.walk(finder) if isinstance(node, ast.Raise)] @@ -466,10 +525,7 @@ def hatari(cg: Configgen) -> list[Launcher]: src.fail("findBestTos is not given a batoceraPaths directory") search_dir = cg.paths[call.args[0].id] kept = _trace(body, index) - options = [ - option[0] for stmt in kept for node in ast.walk(stmt) - if (option := _config_option(node)) is not None - ] + options = list(dict.fromkeys(name for stmt in kept for name in _option_names(stmt))) spans = [src.span(stmt) for stmt in kept] + [(finder.lineno, finder.end_lineno)] args: list[str] = [] second = call.args[1] if len(call.args) > 1 else None @@ -484,6 +540,14 @@ def hatari(cg: Configgen) -> list[Launcher]: args.append("{machine}") elif isinstance(element, ast.Name) and element.id == tos_var: args.append("") + elif ( + isinstance(element, ast.BinOp) + and isinstance(element.right, ast.Name) + and element.right.id == tos_var + and _path_value(element.left, cg.paths) == search_dir + ): + # KNULLI returns the bare name and joins it: BIOS / tos. + args.append("") if "" not in args: src.fail("the TOS findBestTos returns is not put on the command line") @@ -563,7 +627,7 @@ def melonds(cg: Configgen) -> list[Launcher]: node for node in ast.walk(generate) if isinstance(node, ast.Assign) and ast.unparse(node.targets[0]).startswith("base_config['Emu']['ConsoleType']") - and _config_option(node.value) is not None + and _option_names(node.value) ] enabled = [ (key, value) for key, value in zip(base.value.keys, base.value.values) @@ -574,7 +638,7 @@ def melonds(cg: Configgen) -> list[Launcher]: ] if len(console) != 1 or len(enabled) != 1: src.fail("generate no longer sets Emu.ConsoleType from an option with external BIOS on") - option = _config_option(console[0].value)[0] # type: ignore[index] + option = _option_names(console[0].value)[0] # type: ignore[attr-defined] launchers = [] for key, value in zip(base.value.keys, base.value.values): if not isinstance(value, ast.Dict) or not isinstance(key, ast.Constant): @@ -805,7 +869,7 @@ def environment_directories(cg: Configgen) -> list[Launcher]: for module, paths_module in ENVIRONMENT_GENERATORS: src = cg.generator(module) constants = dict(cg.paths) - for node in cg.source(f"{GENERATORS}/{paths_module}").tree.body: + for node in cg.source(f"{cg.generators}/{paths_module}").tree.body: name = _assigned_name(node) value = _path_value(getattr(node, "value", None), constants) if name is not None and value is not None: @@ -840,7 +904,7 @@ FILE_OPTIONS = (("_o2em_options", "o2em_bios"), ("_neocd_options", "neocd_bios") def core_file_options(cg: Configgen) -> list[Launcher]: """_set_from_system(settings, option, system, [name], default=...).""" - src = cg.source(LIBRETRO_OPTIONS) + src = cg.source(cg.libretro_options) table = src.assignment("_option_functions").value if not isinstance(table, ast.Dict): src.fail("_option_functions is not a dict display") From 2c58f50a2d860b984a581f105d26afa71063822b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:47:16 +0100 Subject: [PATCH 2/7] feat: read the knulli launcher layer knulli_launchers reads KNULLI's configgen and the PICO-8 installer at the tag the scraper pins: Batocera's extractors where the generator is still Batocera's code, KNULLI's own for lexaloffle, libretroOptions and knulli-install-pico8, and one AST reader for the BIOS paths the other generators pass on a command line, in a config key or the environment, copy or test for. 52 records. --- platforms/knulli.yml | 638 +++++++++++++++++++++ scripts/scraper/knulli_launchers.py | 839 ++++++++++++++++++++++++++++ scripts/scraper/knulli_scraper.py | 11 +- tests/test_launchers_knulli.py | 523 +++++++++++++++++ 4 files changed, 2008 insertions(+), 3 deletions(-) create mode 100644 scripts/scraper/knulli_launchers.py create mode 100644 tests/test_launchers_knulli.py diff --git a/platforms/knulli.yml b/platforms/knulli.yml index 947545a94..1ca669064 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -5394,3 +5394,641 @@ libretro_systems: - saturn zc210: - zc210 +launchers: +- emulator: duckstation + build: standalone + kind: names_searched + slot: NTSCU + names: + - scph101.bin + - scph1001.bin + - scph5501.bin + - scph7001.bin + - scph7501.bin + case_insensitive: true + search_dir: /userdata/bios + passes_to: "[BIOS] PathNTSCU" + fails_when_missing: No PSX1 BIOS found + fails_scope: emulator + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ + \ 589-611" +- emulator: duckstation + build: standalone + kind: names_searched + slot: PAL + names: + - scph1002.bin + - scph5502.bin + - scph5552.bin + - scph7002.bin + - scph7502.bin + - scph9002.bin + - scph102a.bin + - scph102b.bin + case_insensitive: true + search_dir: /userdata/bios + passes_to: "[BIOS] PathPAL" + fails_when_missing: No PSX1 BIOS found + fails_scope: emulator + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ + \ 589-611" +- emulator: duckstation + build: standalone + kind: names_searched + slot: NTSCJ + names: + - scph100.bin + - scph1000.bin + - scph3000.bin + - scph3500.bin + - scph5500.bin + - scph7000.bin + - scph7003.bin + case_insensitive: true + search_dir: /userdata/bios + passes_to: "[BIOS] PathNTSCJ" + fails_when_missing: No PSX1 BIOS found + fails_scope: emulator + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ + \ 589-611" +- emulator: duckstation + build: standalone + kind: names_searched + slot: Uni + names: + - psxonpsp660.bin + - ps1_rom.bin + case_insensitive: true + search_dir: /userdata/bios + passes_to: "[BIOS] PathNTSCU, PathPAL, PathNTSCJ" + fails_when_missing: No PSX1 BIOS found + fails_scope: emulator + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation/duckstationGenerator.py:119-143,\ + \ 589-611" +- emulator: duckstation-legacy + build: standalone + kind: names_searched + slot: NTSCU + names: + - scph101.bin + - scph1001.bin + - scph5501.bin + - scph7001.bin + - scph7501.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: "[BIOS] PathNTSCU" + fails_when_missing: No PSX1 BIOS found + fails_scope: emulator + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" +- emulator: duckstation-legacy + build: standalone + kind: names_searched + slot: PAL + names: + - scph1002.bin + - scph5502.bin + - scph5552.bin + - scph7002.bin + - scph7502.bin + - scph9002.bin + - scph102a.bin + - scph102b.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: "[BIOS] PathPAL" + fails_when_missing: No PSX1 BIOS found + fails_scope: emulator + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" +- emulator: duckstation-legacy + build: standalone + kind: names_searched + slot: NTSCJ + names: + - scph100.bin + - scph1000.bin + - scph3000.bin + - scph3500.bin + - scph5500.bin + - scph7000.bin + - scph7003.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: "[BIOS] PathNTSCJ" + fails_when_missing: No PSX1 BIOS found + fails_scope: emulator + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/duckstation_legacy/duckstationLegacyGenerator.py:116-153" +- emulator: hatari + build: standalone + kind: names_searched + slot: st + names: + - "etos256{L}.img" + - "tos104{L}.img" + - "tos102{L}.img" + - "tos100{L}.img" + expands: + L: &id001 + - us + - uk + - de + - es + - fr + - it + - nl + - ru + - se + - '' + case_insensitive: false + search_dir: /userdata/bios + passes_to: --machine st --tos + fails_when_missing: no bios found for machine st + depends_on_option: model, language + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/hatari/hatariGenerator.py:33-55,\ + \ 67-78, 172-207" +- emulator: hatari + build: standalone + kind: names_searched + slot: ste + names: + - "etos256{L}.img" + - "tos162{L}.img" + - "tos106{L}.img" + expands: + L: *id001 + case_insensitive: false + search_dir: /userdata/bios + passes_to: --machine ste --tos + fails_when_missing: no bios found for machine ste + depends_on_option: model, language + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/hatari/hatariGenerator.py:33-55,\ + \ 67-78, 172-207" +- emulator: hatari + build: standalone + kind: names_searched + slot: megaste + names: + - "etos256{L}.img" + - "tos206{L}.img" + - "tos205{L}.img" + expands: + L: *id001 + case_insensitive: false + search_dir: /userdata/bios + passes_to: --machine megaste --tos + fails_when_missing: no bios found for machine megaste + depends_on_option: model, language + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/hatari/hatariGenerator.py:33-55,\ + \ 67-78, 172-207" +- emulator: hatari + build: standalone + kind: names_searched + slot: tt + names: + - "etos512{L}.img" + - "tos306{L}.img" + expands: + L: *id001 + case_insensitive: false + search_dir: /userdata/bios + passes_to: --machine tt --tos + fails_when_missing: no bios found for machine tt + depends_on_option: model, language + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/hatari/hatariGenerator.py:33-55,\ + \ 67-78, 172-207" +- emulator: hatari + build: standalone + kind: names_searched + slot: falcon + names: + - "etos512{L}.img" + - "tos404{L}.img" + - "tos402{L}.img" + - "tos400{L}.img" + expands: + L: *id001 + case_insensitive: false + search_dir: /userdata/bios + passes_to: --machine falcon --tos + fails_when_missing: no bios found for machine falcon + depends_on_option: model, language + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/hatari/hatariGenerator.py:33-55,\ + \ 67-78, 172-207" +- emulator: xemu + build: standalone + kind: fixed_name + names: + - cerbios.bin + - Complex_4627.bin + - mcpx_1.0.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: "[sys.files] flashrom_path, [sys.files] bootrom_path" + fails_when_missing: false + depends_on_option: system.name == 'chihiro' + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/xemu/xemuConfig.py:72-78" +- emulator: melonds + build: standalone + kind: fixed_name + names: + - firmware.bin + - bios7.bin + - bios9.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: DS.FirmwarePath, DS.BIOS7Path, DS.BIOS9Path + fails_when_missing: false + depends_on_option: melonds_console + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/melonds/melondsGenerator.py:52-56,\ + \ 74-77, 173" +- emulator: melonds + build: standalone + kind: fixed_name + names: + - dsi_firmware.bin + - bios9.bin + - bios7.bin + - dsi_nand.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: DSi.FirmwarePath, DSi.BIOS9Path, DSi.BIOS7Path, DSi.NANDPath + fails_when_missing: false + depends_on_option: melonds_console + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/melonds/melondsGenerator.py:62-77,\ + \ 173" +- emulator: mupen64plus + build: standalone + kind: fixed_name + names: + - 64DD_IPL.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: "[64DD] IPL-ROM" + fails_when_missing: false + depends_on_option: system.name == 'n64dd' + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mupen/mupenConfig.py:200-204" +- emulator: x16emu + build: standalone + kind: fixed_name + names: + - commanderx16/rom.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: command line -rom + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/x16emu/x16emuGenerator.py:31" +- emulator: flycast + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/dc + passes_to: environment FLYCAST_BIOS_PATH + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/flycast/flycastGenerator.py:239" +- emulator: lexaloffle + build: standalone + kind: fixed_name + slot: pico8 + names: + - pico-8/pico8 + case_insensitive: false + search_dir: /userdata/bios + passes_to: exec + fails_when_missing: Lexaloffle official binary not found at /userdata/bios/pico-8/pico8, + File /userdata/bios/pico-8/pico8 is not set as executable + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/lexaloffle/lexaloffleGenerator.py:20,\ + \ 43-44, 54-59, 140-141" +- emulator: lexaloffle + build: standalone + kind: fixed_name + slot: voxatron + names: + - voxatron/vox + case_insensitive: false + search_dir: /userdata/bios + passes_to: exec + fails_when_missing: Lexaloffle official binary not found at /userdata/bios/voxatron/vox, + File /userdata/bios/voxatron/vox is not set as executable + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/lexaloffle/lexaloffleGenerator.py:27,\ + \ 47-48, 54-59, 140-141" +- emulator: o2em + build: libretro + kind: core_option + names: + - g7400.bin + - o2rom.bin + passes_to: core option o2em_bios + fails_when_missing: false + depends_on_option: "o2em_bios (default by system.name == 'videopacplus')" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroOptions.py:675-676,\ + \ 680-685" +- emulator: neocd + build: libretro + kind: core_option + names: + - "neocd_z.rom (CDZ)" + passes_to: core option neocd_bios + fails_when_missing: false + depends_on_option: neocd_bios + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroOptions.py:2558-2559,\ + \ 2566-2569" +- emulator: lexaloffle + build: standalone + kind: moves + names: + - pico-8/pico8 + - pico-8/pico8_dyn + - pico-8/pico8_gpio + - pico-8/pico8.dat + case_insensitive: false + search_dir: /userdata/bios + passes_to: moved from /userdata/roms/pico8/pico-8*raspi*.zip into /userdata/bios/pico-8 + fails_when_missing: Please add the Raspberry Pi version of Pico-8 to roms/pico8 + and try again. + source_ref: "board/fsoverlay/usr/bin/knulli-install-pico8:3-11, 16-19, 66-69" +- emulator: dolphin + build: standalone + kind: fixed_name + names: + - "GC/{region}/IPL.bin" + expands: + region: + - USA + - EUR + - JAP + case_insensitive: false + search_dir: /userdata/bios + passes_to: "presence sets [Core] SkipIPL = False" + fails_when_missing: false + depends_on_option: "system.isOptSet('dolphin_SkipIPL') and system.getOptBoolean('dolphin_SkipIPL')" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/dolphin/dolphinGenerator.py:192-193,\ + \ 196-197" +- emulator: yabasanshiro + build: standalone + kind: fixed_name + names: + - saturn_bios.bin + case_insensitive: false + search_dir: /userdata/bios + passes_to: command line -b + fails_when_missing: false + depends_on_option: "not (system.isOptSet('yaba_bios_hle') and system.config['yaba_bios_hle']\ + \ == '1')" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/yabasanshiro/yabasanshiroGenerator.py:309-311" +- emulator: rpcs3 + build: standalone + kind: fixed_name + names: + - PS3UPDAT.PUP + case_insensitive: false + search_dir: /userdata/bios + passes_to: command line --installfw + fails_when_missing: false + depends_on_option: "Rpcs3Generator.getFirmwareVersion() is None" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/rpcs3/rpcs3Generator.py:355-358" +- emulator: ryujinx + build: standalone + kind: copies + names: + - switch/prod.keys + case_insensitive: false + search_dir: /userdata/bios + passes_to: copied to /userdata/system/configs/Ryujinx/system/prod.keys + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/ryujinx/ryujinxGenerator.py:107-109" +- emulator: pcsx2 + build: standalone + kind: copies + names: + - ps2/patches.zip + case_insensitive: false + search_dir: /userdata/bios + passes_to: copied from /usr/share/knulli/datainit/bios/ps2/patches.zip + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/pcsx2/pcsx2Generator.py:78,\ + \ 121-122" +- emulator: aethersx2 + build: standalone + kind: copies + names: + - ps2/patches.zip + case_insensitive: false + search_dir: /userdata/bios + passes_to: copied from /usr/share/knulli/datainit/bios/ps2/patches.zip + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/aethersx2/aethersx2Generator.py:59,\ + \ 93-94" +- emulator: gsplus + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios + passes_to: g_cfg_rom_path + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/gsplus/gsplusGenerator.py:106" +- emulator: tsugaru + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/fmtowns + passes_to: command line argument 1 + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/tsugaru/tsugaruGenerator.py:18-19" +- emulator: scummvm + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/scummvm/extra + passes_to: command line --extrapath + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/scummvm/scummvmGenerator.py:117-123" +- emulator: amiberry + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/amiga/ + passes_to: rom_path + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/amiberry/amiberryGenerator.py:40" +- emulator: fsuae + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/amiga + passes_to: command line --kickstarts_dir + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/fsuae/fsuaeGenerator.py:63-69" +- emulator: mame + build: standalone + kind: passes_path + names: [] + search_dir: ;/userdata/bios/mame;/userdata/bios + passes_to: command line -rompath + fails_when_missing: false + depends_on_option: messMode == -1 + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:116-117" +- emulator: mame + build: standalone + kind: passes_path + names: [] + search_dir: ;/userdata/bios/mame;/userdata/bios;/userdata/roms/mame; + passes_to: command line -rompath + fails_when_missing: false + depends_on_option: "not (messMode == -1) and softList in subdirSoftList" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:116,\ + \ 119-120" +- emulator: mame + build: standalone + kind: passes_path + names: [] + search_dir: ;/userdata/bios/mame;/userdata/bios;/userdata/roms/mame + passes_to: command line -rompath + fails_when_missing: false + depends_on_option: "not (messMode == -1) and not (softList in subdirSoftList)" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:116,\ + \ 119, 122" +- emulator: mame + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/mame/samples + passes_to: command line -samplepath + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:129" +- emulator: mame + build: standalone + kind: passes_path + names: [] + search_dir: /var/run/mame_artwork/;/usr/bin/mame/artwork/;/userdata/bios/mame/artwork;/userdata/decorations + passes_to: command line -artpath + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:130" +- emulator: mame + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/mame/artwork/crosshairs + passes_to: command line -crosshairpath + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:180" +- emulator: openmsx + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/Machines + passes_to: "written: filepool add -path /userdata/bios/Machines -types system_rom\ + \ -position 1" + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/openmsx/openmsxGenerator.py:95" +- emulator: openmsx + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/openmsx + passes_to: "written: filepool add -path /userdata/bios/openmsx -types system_rom\ + \ -position 2" + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/openmsx/openmsxGenerator.py:96" +- emulator: fpinball + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios + passes_to: environment W_CACHE + fails_when_missing: false + depends_on_option: "not wsh57_done.exists()" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/fpinball/fpinballGenerator.py:37,\ + \ 39-43" +- emulator: retroarch + build: libretro + kind: passes_path + names: [] + search_dir: /userdata/bios/ + passes_to: system_directory + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroRetroarchCustom.py:98" +- emulator: puae, puae2021, uae4arm + build: libretro + kind: passes_path + names: [] + search_dir: /userdata/bios/amiga/ + passes_to: system_directory + fails_when_missing: false + depends_on_option: "system.config['core'] == 'puae' or system.config['core'] ==\ + \ 'puae2021' or system.config['core'] == 'uae4arm'" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroConfig.py:293-295" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: passes_path + names: [] + search_dir: ;/userdata/bios/mame/;/userdata/bios/ + passes_to: command line -rompath + fails_when_missing: false + depends_on_option: "system.name in ['mame', 'neogeo', 'lcdgames', 'plugnplay', 'vis']" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:64,\ + \ 77" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: passes_path + names: [] + search_dir: ;/userdata/bios/ + passes_to: command line -rompath + fails_when_missing: false + depends_on_option: "not (system.name in ['mame', 'neogeo', 'lcdgames', 'plugnplay',\ + \ 'vis']) and messSysName[messMode] == ''" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:64,\ + \ 123, 132" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: passes_path + names: [] + search_dir: ;/userdata/bios/ + passes_to: command line -rompath + fails_when_missing: false + depends_on_option: "not (system.name in ['mame', 'neogeo', 'lcdgames', 'plugnplay',\ + \ 'vis']) and not (messSysName[messMode] == '') and softList != ''" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:64,\ + \ 123, 197, 204" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: passes_path + names: [] + search_dir: ;/userdata/bios/ + passes_to: command line -rompath + fails_when_missing: false + depends_on_option: "not (system.name in ['mame', 'neogeo', 'lcdgames', 'plugnplay',\ + \ 'vis']) and not (messSysName[messMode] == '') and not (softList != '')" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:64,\ + \ 123, 197, 263" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: passes_path + names: [] + search_dir: /var/run/mame_artwork/;/usr/bin/mame/artwork/;/userdata/bios/lr-mame/artwork;/userdata/bios/mame/artwork;/userdata/decorations + passes_to: command line -artpath + fails_when_missing: false + depends_on_option: "system.config['core'] != 'same_cdi' and not (system.isOptSet('sharemameart')\ + \ and (not system.getOptBoolean('sharemameart'))) and not system.name == 'ti99'" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:412-416,\ + \ 419-420" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: passes_path + names: [] + search_dir: /var/run/mame_artwork/;/usr/bin/mame/artwork/;/userdata/bios/lr-mame/artwork + passes_to: command line -artpath + fails_when_missing: false + depends_on_option: "system.config['core'] != 'same_cdi' and not (not (system.isOptSet('sharemameart')\ + \ and (not system.getOptBoolean('sharemameart')))) and not system.name == 'ti99'" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:412-415,\ + \ 418-420" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: passes_path + names: [] + search_dir: /userdata/bios/mame/samples + passes_to: command line -samplepath + fails_when_missing: false + depends_on_option: not system.name == 'ti99' + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:431-432,\ + \ 435" diff --git a/scripts/scraper/knulli_launchers.py b/scripts/scraper/knulli_launchers.py new file mode 100644 index 000000000..8ebc0fd59 --- /dev/null +++ b/scripts/scraper/knulli_launchers.py @@ -0,0 +1,839 @@ +"""KNULLI's launcher layer, read from its configgen and its board scripts. + +KNULLI is built on Batocera and keeps configgen under its own package +(package/system/knulli-configgen). Where a generator is still Batocera's +code (DuckStation, xemu, mupen64plus, x16emu, the flycast environment, the +hatari and melonDS forms batocera_launchers accepts), Batocera's extractor +reads it. The others are older or KNULLI's own: the lexaloffle generator logs +and returns instead of raising, libretroOptions is one function of +`if (system.config['core'] == ...)` blocks, and the PICO-8 binary is +installed by board/fsoverlay/usr/bin/knulli-install-pico8. + +The remaining generators name a /userdata/bios path in a command line, a +config key, an environment variable, a copy or a presence test. bios_sites() +reads each of those uses with ast and the context that consumes it, and +GENERIC_SITES says, per generator, whether the path is a file or a directory. +A generator in that table that no longer names a BIOS path raises +LauncherShapeError, as every other extractor does. +""" + +from __future__ import annotations + +import ast +import re +from collections.abc import Callable, Iterator +from dataclasses import dataclass, field +from typing import Any + +try: + from .batocera_launchers import ( + Configgen, + _path_value, + _render, + _system_branches, + duckstation, + duckstation_legacy, + environment_directories, + hatari, + melonds, + mupen64plus, + x16emu, + xemu, + ) + from .configgen_launchers import ( + Launcher, + LauncherShapeError, + Source, + _assigned_name, + ) + from .shell_launchers import LauncherPatternMissing, Script, source_ref +except ImportError: + from batocera_launchers import ( # type: ignore[no-redef] + Configgen, + _path_value, + _render, + _system_branches, + duckstation, + duckstation_legacy, + environment_directories, + hatari, + melonds, + mupen64plus, + x16emu, + xemu, + ) + from configgen_launchers import ( # type: ignore[no-redef] + Launcher, + LauncherShapeError, + Source, + _assigned_name, + ) + from shell_launchers import ( # type: ignore[no-redef] + LauncherPatternMissing, + Script, + source_ref, + ) + +CONFIGGEN = "package/system/knulli-configgen/configgen/configgen" +GENERATORS = f"{CONFIGGEN}/generators" +LIBRETRO_OPTIONS = f"{GENERATORS}/libretro/libretroOptions.py" +PICO8_INSTALLER = "board/fsoverlay/usr/bin/knulli-install-pico8" + +# Batocera's extractors that read KNULLI's generators unchanged. +SHARED = ( + duckstation, duckstation_legacy, hatari, xemu, melonds, mupen64plus, x16emu, + environment_directories, +) + + +def _lines(node: ast.AST) -> tuple[int, int]: + """First and last line of a node.""" + first: int = getattr(node, "lineno", 0) + return first, getattr(node, "end_lineno", None) or first + + +def _bound(node: ast.AST) -> ast.expr | None: + """The value of an assignment statement.""" + value = getattr(node, "value", None) + return value if isinstance(value, ast.expr) else None + + +def configgen(read: Callable[[str], str]) -> Configgen: + """KNULLI's configgen: Batocera's reader at KNULLI's package, no legacy map.""" + return Configgen(read, CONFIGGEN, legacy_map=False) + + +def _error_guard( + src: Source, scope: ast.FunctionDef, test: Callable[[ast.expr], bool], what: str +) -> tuple[ast.If, ast.JoinedStr]: + """`if : eslog.error(f"..."); return -1` and its message.""" + found = [] + for node in ast.walk(scope): + if not (isinstance(node, ast.If) and test(node.test) and len(node.body) == 2): + continue + logged, returned = node.body + if ( + isinstance(logged, ast.Expr) + and isinstance(logged.value, ast.Call) + and ast.unparse(logged.value.func) == "eslog.error" + and logged.value.args + and isinstance(logged.value.args[0], ast.JoinedStr) + and isinstance(returned, ast.Return) + ): + found.append((node, logged.value.args[0])) + if len(found) != 1: + src.fail(f"expected one `if {what}: eslog.error(...); return`, found {len(found)}") + return found[0] + + +def lexaloffle(cg: Configgen) -> list[Launcher]: + """Per system, BIN_PATH under /userdata/bios run as the emulator binary.""" + module = "lexaloffle.lexaloffleGenerator" + src = cg.generator(module) + generate = src.function("generate") + defined = { + name: node for node in src.tree.body if (name := _assigned_name(node)) is not None + } + constants = dict(cg.paths) + for name, node in defined.items(): + value = _path_value(_bound(node), constants) + if value is not None: + constants[name] = value + missing, missing_message = _error_guard( + src, generate, + lambda t: isinstance(t, ast.UnaryOp) and isinstance(t.op, ast.Not) + and isinstance(t.operand, ast.Call) and ast.unparse(t.operand.func).endswith(".exists"), + "not .exists()", + ) + binary = ast.unparse(missing.test.operand.func).removesuffix(".exists") # type: ignore[attr-defined] + access, access_message = _error_guard( + src, generate, + lambda t: isinstance(t, ast.UnaryOp) and isinstance(t.op, ast.Not) + and ast.unparse(t.operand).startswith(f"os.access({binary}"), + f"not os.access({binary}, ...)", + ) + commands = [ + node for node in ast.walk(generate) + if isinstance(node, ast.Assign) and isinstance(node.value, ast.List) + and node.value.elts and ast.unparse(node.value.elts[0]) == binary + ] + if len(commands) != 1: + src.fail(f"expected one command line starting with {binary}, found {len(commands)}") + prefix = cg.bios.rstrip("/") + "/" + launchers = [] + for system, branch in _system_branches(generate): + bound = [ + stmt for stmt in branch.body + if _assigned_name(stmt) == binary and isinstance(_bound(stmt), ast.Name) + ] + if len(bound) != 1: + src.fail(f"the {system} branch does not set {binary} once") + constant = ast.unparse(bound[0].value) # type: ignore[attr-defined] + path = constants.get(constant) + if path is None or not path.startswith(prefix): + src.fail(f"{constant} is not a path under {cg.bios}") + messages = [_render(m, binary, path) for m in (missing_message, access_message)] + launchers.append(Launcher( + emulator=cg.emulator(module), build="standalone", kind="fixed_name", + slot=system, names=[path[len(prefix):]], case_insensitive=False, + search_dir=cg.bios, passes_to="exec", + fails_when_missing=", ".join(messages), + source_ref=src.ref([ + src.span(defined[constant]), src.span(branch.test), src.span(bound[0]), + _lines(missing), _lines(access), + src.span(commands[0]), + ]), + )) + if not launchers: + src.fail("generate no longer branches on system.name") + return launchers + + +def pico8_installer(read: Callable[[str], str], bios: str) -> list[Launcher]: + """knulli-install-pico8: unzips the Raspberry Pi build and moves it into the BIOS folder.""" + script = Script(PICO8_INSTALLER, read(PICO8_INSTALLER)) + lines: set[int] = set() + archive_line, archive = script.find(r"\$\(ls -d (\S+\.zip)", "the PICO-8 zip lookup") + lines.add(archive_line) + guard_line, _ = script.find(r'^if \[ -z \$PICO_8_ZIPFILE \]', "the missing zip test") + message_line, message = script.find(r'^\s*echo "([^"]+)"', "the missing zip message", guard_line) + lines |= {guard_line, message_line} + names: list[str] = [] + prefix = bios.rstrip("/") + "/" + for number, line in script.code(): + moved = re.match(r"\s*mv\s+(\S+)\s+(\S+)\s*$", line) + if moved is None: + continue + used: set[int] = set() + source = script.expand(moved.group(1), number, used) + target = script.expand(moved.group(2).strip('"'), number, used) + if not target.startswith(prefix): + continue + if target.endswith("/"): + target += source.rsplit("/", 1)[-1] + names.append(target[len(prefix):]) + lines |= used | {number} + if not names: + raise script.missing(f"mv into {bios}") + folder = names[0].rsplit("/", 1)[0] + return [Launcher( + emulator="lexaloffle", build="standalone", kind="moves", names=names, + case_insensitive=False, search_dir=bios, + passes_to=f"moved from {archive.group(1)} into {prefix}{folder}", + fails_when_missing=message.group(1), + source_ref=source_ref([(PICO8_INSTALLER, lines)]), + )] + + +def _quoted(node: ast.AST) -> str | None: + """The value of a '"name"' core option literal.""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + value = node.value + if len(value) >= 2 and value[0] == value[-1] == '"': + return value[1:-1] + return None + + +def _cores_of(test: ast.expr) -> list[str]: + """Core names `system.config['core'] == 'x'`, `in [...]` or an `or` of them tests.""" + if isinstance(test, ast.BoolOp) and isinstance(test.op, ast.Or): + return [core for value in test.values for core in _cores_of(value)] + if ( + isinstance(test, ast.Compare) + and ast.unparse(test.left) == "system.config['core']" + and len(test.ops) == 1 + ): + right = test.comparators[0] + if isinstance(test.ops[0], ast.Eq) and isinstance(right, ast.Constant): + return [str(right.value)] + if isinstance(test.ops[0], ast.In) and isinstance(right, (ast.List, ast.Tuple)): + return [str(e.value) for e in right.elts if isinstance(e, ast.Constant)] + return [] + + +# libretro core options whose value names the BIOS file the core opens, +# the same two batocera_launchers.FILE_OPTIONS reads. +FILE_OPTIONS = ("o2em_bios", "neocd_bios") + + +def core_file_options(cg: Configgen) -> list[Launcher]: + """if isOptSet(o): save(o, option) elif ...: save(o, '"a"') else: save(o, '"b"').""" + src = cg.source(LIBRETRO_OPTIONS) + function = src.function("generateCoreSettings") + launchers = [] + for setting in FILE_OPTIONS: + chains = [ + node for node in ast.walk(function) + if isinstance(node, ast.If) + and ast.unparse(node.test) == f"system.isOptSet({setting!r})" + ] + if len(chains) != 1: + src.fail(f"expected one `if system.isOptSet({setting!r})`, found {len(chains)}") + chain = chains[0] + names: list[str] = [] + conditions: list[str] = [] + branch: ast.If | None = chain + tail: list[ast.stmt] = chain.orelse + while branch is not None: + tail = branch.orelse + if branch is not chain: + saved = _saved(branch.body, setting) + if saved is None: + src.fail(f"a branch of {setting} saves no literal name") + names.append(saved) + conditions.append(ast.unparse(branch.test)) + branch = tail[0] if len(tail) == 1 and isinstance(tail[0], ast.If) else None + default = _saved(tail, setting) + if default is None: + src.fail(f"{setting} has no literal default") + names.append(default) + holder = next( + ( + node for node in ast.walk(function) + if isinstance(node, ast.If) and _cores_of(node.test) + and any(child is chain for child in ast.walk(node)) + ), + None, + ) + if holder is None: + src.fail(f"{setting} is not set under a core test") + depends = setting if not conditions else f"{setting} (default by {', '.join(conditions)})" + launchers.append(Launcher( + emulator=", ".join(_cores_of(holder.test)), build="libretro", kind="core_option", + names=names, passes_to=f"core option {setting}", depends_on_option=depends, + source_ref=src.ref([src.span(holder.test), _lines(chain)]), + )) + return launchers + + +def _saved(body: list[ast.stmt], setting: str) -> str | None: + """The literal a one-statement body saves under setting.""" + if len(body) != 1 or not isinstance(body[0], ast.Expr): + return None + call = body[0].value + if ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "save" + and len(call.args) == 2 + and isinstance(call.args[0], ast.Constant) + and call.args[0].value == setting + ): + return _quoted(call.args[1]) + return None + + +# --------------------------------------------------------------------------- +# Generic reading of the /userdata/bios paths a generator hands over. + + +@dataclass(frozen=True) +class GenericSites: + """A generator module whose BIOS path uses bios_sites() reads. + + kind is fixed_name when the path names a file, passes_path when it names + a directory. A copy is recorded as `copies` whatever the kind. caller + names the module and function whose core test selects the cores this + helper module runs for, emulator the name when no core test does. + """ + + path: str + kind: str + build: str = "standalone" + emulator: str | None = None + caller: tuple[str, str] | None = None + + +# The names configgen gives the command it builds. +COMMAND_NAMES = {"commandArray", "commandLine"} + +GENERIC_SITES = ( + GenericSites("dolphin/dolphinGenerator.py", "fixed_name"), + GenericSites("yabasanshiro/yabasanshiroGenerator.py", "fixed_name"), + GenericSites("rpcs3/rpcs3Generator.py", "fixed_name"), + GenericSites("ryujinx/ryujinxGenerator.py", "fixed_name"), + GenericSites("pcsx2/pcsx2Generator.py", "fixed_name"), + GenericSites("aethersx2/aethersx2Generator.py", "fixed_name"), + GenericSites("gsplus/gsplusGenerator.py", "passes_path"), + GenericSites("tsugaru/tsugaruGenerator.py", "passes_path"), + GenericSites("scummvm/scummvmGenerator.py", "passes_path"), + GenericSites("amiberry/amiberryGenerator.py", "passes_path"), + GenericSites("fsuae/fsuaeGenerator.py", "passes_path"), + GenericSites("mame/mameGenerator.py", "passes_path"), + GenericSites("openmsx/openmsxGenerator.py", "passes_path"), + GenericSites("fpinball/fpinballGenerator.py", "passes_path"), + GenericSites( + "libretro/libretroRetroarchCustom.py", "passes_path", "libretro", emulator="retroarch" + ), + GenericSites("libretro/libretroConfig.py", "passes_path", "libretro"), + GenericSites( + "libretro/libretroMAMEConfig.py", "passes_path", "libretro", + caller=("libretro/libretroConfig.py", "generateMAMEConfigs"), + ), +) + + +@dataclass +class _Site: + """One use of a BIOS path, and the record fields it contributes.""" + + path: str + passes_to: str + kind: str | None + lines: list[tuple[int, int]] + conditions: list[str] + cores: list[str] + expands: dict[str, list[str]] = field(default_factory=dict) + + +class _Reader: + """The BIOS path uses of one generator module.""" + + def __init__(self, cg: Configgen, src: Source) -> None: + self.cg = cg + self.src = src + self.bios = cg.bios.rstrip("/") + self.parents: dict[int, ast.AST] = {} + for parent in ast.walk(src.tree): + for child in ast.iter_child_nodes(parent): + self.parents[id(child)] = parent + self.constants = dict(cg.paths) + folder = src.path.rsplit("/", 1)[0] + for node in src.tree.body: + if ( + isinstance(node, ast.ImportFrom) + and node.level == 1 + and node.module + and node.module.endswith("Paths") + ): + paths = cg.source(f"{folder}/{node.module}.py") + self._define(paths.tree.body) + self._define(src.tree.body) + + def _define(self, body: list[ast.stmt]) -> None: + for node in body: + name = _assigned_name(node) + value = self.resolve(getattr(node, "value", None)) + if name is not None and value is not None: + self.constants[name] = value + + def under_bios(self, value: str | None) -> bool: + return value is not None and ( + value.rstrip("/") == self.bios or f"{self.bios}/" in value + ) + + def resolve(self, node: ast.AST | None) -> str | None: + """The path an expression spells, unknown parts of an f-string as .""" + if node is None: + return None + value = _path_value(node, self.constants) + if value is not None: + return value + if isinstance(node, ast.Constant) and isinstance(node.value, str): + text = node.value.strip('"') + return text if text.startswith("/") else None + if isinstance(node, ast.Call) and ast.unparse(node.func) == "str" and len(node.args) == 1: + return self.resolve(node.args[0]) + if isinstance(node, ast.JoinedStr): + parts = [] + for part in node.values: + if isinstance(part, ast.Constant): + parts.append(str(part.value)) + else: + inner = self.resolve(part.value) # type: ignore[attr-defined] + parts.append(inner if inner is not None else f"<{ast.unparse(part.value)}>") # type: ignore[attr-defined] + return "".join(parts).strip('"') + if ( + isinstance(node, ast.BinOp) + and isinstance(node.op, ast.Div) + and isinstance(node.right, (ast.Constant, ast.Name)) + ): + left = self.resolve(node.left) + right = node.right.value if isinstance(node.right, ast.Constant) else None + if isinstance(node.right, ast.Name): + right = self._placeholder(node.right) + if left is not None and isinstance(right, str): + return f"{left.rstrip('/')}/{right}" + return None + + def _placeholder(self, name: ast.Name) -> str | None: + """{var} for a comprehension variable over a literal list.""" + comprehension = self._comprehension(name) + if comprehension is None: + return None + return "{" + name.id + "}" + + def _comprehension(self, name: ast.Name) -> tuple[str, list[str]] | None: + node: ast.AST | None = name + while node is not None: + node = self.parents.get(id(node)) + if isinstance(node, (ast.GeneratorExp, ast.ListComp)): + for generator in node.generators: + if isinstance(generator.target, ast.Name) and generator.target.id == name.id: + values = self._literal_list(generator.iter, node) + return (name.id, values) if values else None + return None + + def _literal_list(self, node: ast.AST, scope_of: ast.AST) -> list[str] | None: + if isinstance(node, ast.List): + return [str(e.value) for e in node.elts if isinstance(e, ast.Constant)] + if isinstance(node, ast.Name): + scope = self.function_of(scope_of) + bound = [ + n for n in ast.walk(scope or self.src.tree) + if _assigned_name(n) == node.id and isinstance(_bound(n), ast.List) + ] + if len(bound) == 1: + return self._literal_list(_bound(bound[0]), scope_of) # type: ignore[arg-type] + return None + + def function_of(self, node: ast.AST) -> ast.FunctionDef | None: + while node is not None: + node = self.parents.get(id(node)) # type: ignore[assignment] + if isinstance(node, ast.FunctionDef): + return node + return None + + def statement_of(self, node: ast.AST) -> ast.stmt: + while not isinstance(node, ast.stmt): + node = self.parents[id(node)] + return node + + def maximal(self) -> Iterator[tuple[ast.AST, str]]: + """Each expression spelling a BIOS path that no larger one contains.""" + for node in ast.walk(self.src.tree): + if not isinstance(node, ast.expr): + continue + value = self.resolve(node) + if value is None and isinstance(node, ast.Constant) and isinstance(node.value, str): + # A line written to a config file: openmsx's settings.tcl. + written = re.search(rf"{re.escape(self.bios)}(?:/[^\s\"';]*)?", node.value) + parent = self.parents.get(id(node)) + if written and isinstance(parent, ast.Call) and ast.unparse(parent.func).endswith(".write"): + yield node, written.group(0) + continue + if not self.under_bios(value): + continue + parent = self.parents.get(id(node)) + if isinstance(parent, (ast.FormattedValue, ast.JoinedStr)): + continue + if isinstance(parent, ast.expr) and self.under_bios(self.resolve(parent)): + continue + if isinstance(parent, ast.Call) and ast.unparse(parent.func) == "str": + continue + yield node, value # type: ignore[misc] + + def conditions(self, node: ast.AST) -> tuple[list[str], list[tuple[int, int]]]: + """Tests of the ifs a node sits in, outermost first, and their lines. + + A presence test of a BIOS path is a guard, kept for its lines only. + """ + tests: list[str] = [] + spans: list[tuple[int, int]] = [] + child, parent = node, self.parents.get(id(node)) + while parent is not None and not isinstance(parent, ast.FunctionDef): + if isinstance(parent, ast.If) and child is not parent.test: + spans.append(self.src.span(parent.test)) + text = ast.unparse(parent.test) + if not self._is_guard(parent.test): + in_else = any(child is stmt for stmt in parent.orelse) + tests.append(f"not ({text})" if in_else else text) + child, parent = parent, self.parents.get(id(parent)) + return list(reversed(tests)), spans + + def _is_guard(self, test: ast.expr) -> bool: + for n in ast.walk(test): + if not ( + isinstance(n, ast.Call) + and isinstance(n.func, ast.Attribute) + and n.func.attr == "exists" + ): + continue + receiver = n.func.value + if self.under_bios(self.resolve(receiver)): + return True + if isinstance(receiver, ast.Name) and self.local_value(receiver) is not None: + return True + return False + + def cores(self, node: ast.AST) -> list[str]: + child, parent = node, self.parents.get(id(node)) + while parent is not None: + if isinstance(parent, ast.If) and child is not parent.test and _cores_of(parent.test): + return _cores_of(parent.test) + child, parent = parent, self.parents.get(id(parent)) + return [] + + def uses(self, node: ast.AST, value: str) -> Iterator[tuple[ast.AST, str | None, str | None]]: + """(node read, passes_to, kind) for each consumer of a BIOS path expression. + + A path bound to a local name is followed to each read of that name. + A use with no consumer (a presence test, a mkdir) yields passes_to None. + """ + parent = self.parents.get(id(node)) + if isinstance(parent, ast.Assign) and len(parent.targets) == 1: + target = parent.targets[0] + if isinstance(target, ast.Name): + scope = self.function_of(parent) + if scope is None: + return + for read in ast.walk(scope): + if ( + isinstance(read, ast.Name) + and read.id == target.id + and isinstance(read.ctx, ast.Load) + and read.lineno > parent.lineno + ): + yield from self.uses(read, value) + return + if isinstance(target, ast.Subscript) and isinstance(target.slice, ast.Constant): + yield node, str(target.slice.value), None + return + yield node, *self.consumer(node, value) + + def _command_line(self, display: ast.List | ast.Tuple) -> bool: + """Whether a list display is (part of) the command configgen runs. + + It is assigned or added to commandArray (or commandLine for the + libretro MAME cmd file), or it starts with an absolute binary path. + """ + first = display.elts[0] if display.elts else None + if isinstance(first, ast.Constant) and str(first.value).startswith("/"): + return True + holder = self.parents.get(id(display)) + targets: list[ast.expr] = [] + if isinstance(holder, ast.Assign): + targets = holder.targets + elif isinstance(holder, ast.AugAssign): + targets = [holder.target] + elif ( + isinstance(holder, ast.Call) + and isinstance(holder.func, ast.Attribute) + and holder.func.attr in ("extend", "append") + ): + targets = [holder.func.value] + names = { + ast.unparse(t.value if isinstance(t, ast.Subscript) else t) for t in targets + } + return bool(names & COMMAND_NAMES) + + def local_value(self, name: ast.Name) -> str | None: + """The BIOS path a local name is bound to once in its function.""" + scope = self.function_of(name) + if scope is None: + return None + bound = [ + self.resolve(_bound(n)) + for n in ast.walk(scope) if _assigned_name(n) == name.id + ] + return bound[0] if len(bound) == 1 and self.under_bios(bound[0]) else None + + def consumer(self, node: ast.AST, value: str) -> tuple[str | None, str | None]: + parent = self.parents.get(id(node)) + if isinstance(parent, ast.Attribute): + return None, None + if isinstance(parent, (ast.List, ast.Tuple)): + holder = self.parents.get(id(parent)) + if isinstance(holder, (ast.For, ast.comprehension)): + return None, None + if not self._command_line(parent): + return None, None + index = next(i for i, e in enumerate(parent.elts) if e is node) + if isinstance(node, ast.JoinedStr) and isinstance(node.values[0], ast.Constant): + flag = str(node.values[0].value) + if flag.startswith("-") and flag.endswith("=") and value.startswith(flag): + return f"command line {flag[:-1]}", None + previous = parent.elts[index - 1] if index else None + if isinstance(previous, ast.Constant) and str(previous.value).startswith("-"): + return f"command line {previous.value!s}", None + if index: + return f"command line argument {index}", None + return None, None + if isinstance(parent, ast.Call): + func = parent.func + attr = func.attr if isinstance(func, ast.Attribute) else None + consts = [a.value for a in parent.args if isinstance(a, ast.Constant)] + if attr == "save" and len(parent.args) == 2 and parent.args[1] is node and consts: + return str(consts[0]), None + if attr == "set" and len(parent.args) == 3 and parent.args[2] is node and len(consts) >= 2: + return f"[{consts[0]!s}] {consts[1]!s}", None + receiver = func.value if isinstance(func, ast.Attribute) else None + if attr in ("copy", "copyfile", "copy2") and receiver is not None and ast.unparse(receiver) == "shutil": + source, target = parent.args[0], parent.args[1] + if source is node: + return f"copied to {self.resolve(target) or ast.unparse(target)}", "copies" + return f"copied from {self.resolve(source) or ast.unparse(source)}", "copies" + if ( + attr == "append" + and isinstance(receiver, ast.Name) + and receiver.id in COMMAND_NAMES + and isinstance(node, ast.JoinedStr) + and isinstance(node.values[0], ast.Constant) + ): + flag = str(node.values[0].value) + if flag.startswith("-") and flag.endswith("=") and value.startswith(flag): + return f"command line {flag[:-1]}", None + if attr == "write" and isinstance(node, ast.Constant): + return f"written: {str(node.value).strip()}", None + return None, None + if isinstance(parent, ast.Dict): + key = next( + (k for k, v in zip(parent.keys, parent.values) if v is node), None + ) + if isinstance(key, ast.Constant): + holder = self.parents.get(id(parent)) + is_env = ( + isinstance(holder, ast.keyword) and holder.arg == "env" + ) or _assigned_name(holder) == "env" # type: ignore[arg-type] + return (f"environment {key.value!s}" if is_env else str(key.value)), None + return None, None + + def sites(self) -> list[_Site]: + found: list[_Site] = [] + guards: list[tuple[ast.AST, str]] = [] + for node, value in self.maximal(): + if isinstance(self.statement_of(node), (ast.Import, ast.ImportFrom)): + continue + if self.function_of(node) is None and isinstance(self.parents.get(id(node)), ast.Assign): + continue + for read, passes_to, kind in self.uses(node, value): + parent = self.parents.get(id(read)) + if ( + passes_to is None + and isinstance(parent, ast.Attribute) + and parent.attr == "exists" + ): + guards.append((read, value)) + if passes_to is None: + continue + found.append(self._site(read, value, passes_to, kind, node)) + for read, value in guards: + if any(site.path == value for site in found): + site = next(s for s in found if s.path == value) + site.lines.append(self.src.span(self.statement_of(read))) + continue + found.append(self._guard_site(read, value)) + return sorted(found, key=lambda site: site.lines[0]) + + def _site( + self, read: ast.AST, value: str, passes_to: str, kind: str | None, defined: ast.AST + ) -> _Site: + tests, spans = self.conditions(read) + lines = [self.src.span(self.statement_of(read)), *spans] + if defined is not read: + # A path bound in one branch and read after the if: the branch + # that bound it is a condition of the read. + bound_tests, bound_spans = self.conditions(defined) + tests = list(dict.fromkeys([*bound_tests, *tests])) + lines += [self.src.span(self.statement_of(defined)), *bound_spans] + return _Site(value, passes_to, kind, lines, tests, self.cores(read), self._expands(read)) + + def _guard_site(self, read: ast.AST, value: str) -> _Site: + """A presence test whose if body is the only consumer of the path.""" + test = self.statement_of(read) + if not isinstance(test, ast.If): + self.src.fail(f"line {test.lineno} tests a BIOS path outside an if") + first = test.body[0] + described = ast.unparse(first) + call = first.value if isinstance(first, ast.Expr) else None + if isinstance(call, ast.Call) and isinstance(call.func, ast.Attribute) and call.func.attr == "set": + literal = [a.value for a in call.args if isinstance(a, ast.Constant)] + if len(literal) == 3: + described = f"[{literal[0]!s}] {literal[1]!s} = {literal[2]!s}" + tests, spans = self.conditions(test) + return _Site( + value, f"presence sets {described}", None, + [(test.lineno, first.end_lineno), *spans], # type: ignore[list-item] + tests, self.cores(test), self._expands(read), + ) + + def _expands(self, node: ast.AST) -> dict[str, list[str]]: + found: dict[str, list[str]] = {} + for name in ast.walk(node): + if isinstance(name, ast.Name): + bound = self._comprehension(name) + if bound is not None: + found[bound[0]] = bound[1] + return found + + +def _caller_cores(cg: Configgen, caller: tuple[str, str]) -> list[str]: + path, function = caller + src = cg.source(f"{cg.generators}/{path}") + for node in ast.walk(src.tree): + if not isinstance(node, ast.If) or not _cores_of(node.test): + continue + if any( + isinstance(n, ast.Call) and ast.unparse(n.func).endswith(f".{function}") + for stmt in node.body for n in ast.walk(stmt) + ): + return _cores_of(node.test) + src.fail(f"no core test calls {function}") + + +def bios_sites(cg: Configgen, spec: GenericSites) -> list[Launcher]: + """One record per BIOS path and consumer of a generator module.""" + src = cg.source(f"{cg.generators}/{spec.path}") + reader = _Reader(cg, src) + sites = reader.sites() + if not sites: + src.fail("hands over no /userdata/bios path") + module = spec.path.removesuffix(".py").replace("/", ".") + caller_cores = _caller_cores(cg, spec.caller) if spec.caller else [] + grouped: dict[tuple[str, str, str, tuple[str, ...]], list[_Site]] = {} + for site in sites: + flag = site.passes_to.removeprefix("command line ") + if site.path.startswith(f"{flag}="): + site.path = site.path[len(flag) + 1:] + emulator = ( + spec.emulator + or ", ".join(site.cores or caller_cores) + or (cg.emulator(module) if spec.build == "standalone" else "") + ) + if not emulator: + src.fail(f"no core runs the {site.passes_to} use at line {site.lines[0][0]}") + key = (emulator, site.path, site.passes_to, tuple(site.conditions)) + grouped.setdefault(key, []).append(site) + prefix = cg.bios.rstrip("/") + "/" + launchers = [] + for (emulator, path, passes_to, _conditions), group in grouped.items(): + kind = group[0].kind or spec.kind + conditions = list(dict.fromkeys(c for site in group for c in site.conditions)) + expands = {k: v for site in group for k, v in site.expands.items()} + names: list[str] = [] + search_dir = path + if kind in ("fixed_name", "copies"): + if not path.startswith(prefix): + src.fail(f"{path} names no file under {cg.bios}") + names, search_dir = [path[len(prefix):]], cg.bios + launchers.append(Launcher( + emulator=emulator, build=spec.build, kind=kind, names=names, + expands=expands or None, + case_insensitive=False if names else None, + search_dir=search_dir, passes_to=passes_to, fails_when_missing=False, + depends_on_option=" and ".join(conditions) or None, + source_ref=src.ref([span for site in group for span in site.lines]), + )) + return launchers + + +EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = ( + *SHARED, lexaloffle, core_file_options, +) + + +def extract_launchers(read: Callable[[str], str]) -> list[dict[str, Any]]: + """Every launcher record, read through `read(repo-relative path)`.""" + cg = configgen(read) + launchers = [launcher for extractor in EXTRACTORS for launcher in extractor(cg)] + launchers += pico8_installer(read, cg.bios) + for spec in GENERIC_SITES: + launchers += bios_sites(cg, spec) + return [launcher.entry() for launcher in launchers] + + +__all__ = [ + "CONFIGGEN", "GENERIC_SITES", "LauncherPatternMissing", "LauncherShapeError", + "bios_sites", "configgen", "core_file_options", "extract_launchers", "lexaloffle", + "pico8_installer", +] diff --git a/scripts/scraper/knulli_scraper.py b/scripts/scraper/knulli_scraper.py index 7c2936902..47af8e50c 100644 --- a/scripts/scraper/knulli_scraper.py +++ b/scripts/scraper/knulli_scraper.py @@ -19,9 +19,10 @@ Verification (knulli-systems at the release tag): 10834d00, es-app/src/ApiSystem.cpp:764-774) and by knulli-support:69. es_systems.yml and configgen-defaults.yml have Batocera's shape, so the cores -are read with Batocera's code. The configgen generators do not: the hatari, -melonds and gsplus generators and libretroOptions.py are older forms than the -ones batocera_launchers reads, so no launcher layer is written. +are read with Batocera's code. The launcher layer comes from KNULLI's own +configgen package and board scripts at the same tag (knulli_launchers): +Batocera's extractors where a generator is still Batocera's code, KNULLI's +where it is older or its own. """ from __future__ import annotations @@ -44,6 +45,7 @@ from .batocera_scraper import ( requirements_from_systems, ) from .es_emulators import attach_system_cores +from .knulli_launchers import extract_launchers PLATFORM_NAME = "knulli" @@ -124,6 +126,9 @@ class Scraper(BaseScraper): # configgen hands a libretro core /userdata/saves/, as # Batocera's does: the systems each core runs for are its save folders. result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped) + result["launchers"] = extract_launchers( + lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path) + ) return result diff --git a/tests/test_launchers_knulli.py b/tests/test_launchers_knulli.py new file mode 100644 index 000000000..8ea3070b0 --- /dev/null +++ b/tests/test_launchers_knulli.py @@ -0,0 +1,523 @@ +"""KNULLI's launcher layer, read from its configgen and board scripts. + +The fixtures are cut-down copies of the blocks each extractor reads at +knulli-linux 20260511. Records must carry the names in the order the code +writes or tries them, cite the fixture's own lines, and a construct that is +gone must stop the scrape instead of returning nothing. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import sys +import textwrap +import unittest +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from scraper import knulli_launchers as kl # noqa: E402 +from scraper.batocera_launchers import hatari, melonds # noqa: E402 +from scraper.configgen_launchers import LauncherShapeError # noqa: E402 + +G = kl.GENERATORS + + +def text(block: str) -> str: + return textwrap.dedent(block).lstrip("\n") + + +PATHS = text( + """ + from pathlib import Path + from typing import Final + + KNULLI_SHARE_DIR: Final = Path('/usr/share/knulli') + DATAINIT_DIR: Final = KNULLI_SHARE_DIR / 'datainit' + USERDATA: Final = Path('/userdata') + HOME: Final = USERDATA / 'system' + CONFIGS: Final = HOME / 'configs' + BIOS: Final = USERDATA / 'bios' + ROMS: Final = USERDATA / 'roms' + """ +) + +IMPORTER = text( + """ + from typing import Final + + _GENERATOR_MAP: Final[dict[str, tuple[str, str]]] = { + 'lexaloffle': ('lexaloffle.lexaloffleGenerator', 'LexaloffleGenerator'), + 'yabasanshiro': ('yabasanshiro.yabasanshiroGenerator', 'YabasanshiroGenerator'), + 'ryujinx': ('ryujinx.ryujinxGenerator', 'RyujinxGenerator'), + 'dolphin': ('dolphin.dolphinGenerator', 'DolphinGenerator'), + 'scummvm': ('scummvm.scummvmGenerator', 'ScummVMGenerator'), + 'openmsx': ('openmsx.openmsxGenerator', 'OpenmsxGenerator'), + 'hatari': ('hatari.hatariGenerator', 'HatariGenerator'), + 'melonds': ('melonds.melondsGenerator', 'MelonDSGenerator'), + } + """ +) + +# lexaloffleGenerator.py: the binary constants, the system branches, the two +# checks that log and return, and the command. +LEXALOFFLE = text( + """ + from ...batoceraPaths import BIOS, HOME, ROMS + + PICO8_BIN_PATH: Final = BIOS / "pico-8" / "pico8" + VOX_BIN_PATH: Final = BIOS / "voxatron" / "vox" + + class LexaloffleGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + if (system.name == "pico8"): + BIN_PATH=PICO8_BIN_PATH + elif (system.name == "voxatron"): + BIN_PATH=VOX_BIN_PATH + else: + return -1 + if not BIN_PATH.exists(): + eslog.error(f"Lexaloffle official binary not found at {BIN_PATH}") + return -1 + if not os.access(BIN_PATH, os.X_OK): + eslog.error(f"File {BIN_PATH} is not set as executable") + return -1 + commandArray = [BIN_PATH] + return Command.Command(array=commandArray, env={}) + """ +) + +# libretroOptions.py: one function of core blocks, the option or a literal. +OPTIONS = text( + """ + def generateCoreSettings(coreSettings, system, rom, guns, wheels): + if (system.config['core'] == 'o2em'): + coreSettings.save('o2em_vkbd_transparency', '"25"') + if system.isOptSet('o2em_bios'): + coreSettings.save('o2em_bios', '"' + system.config['o2em_bios'] + '"') + elif system.name == 'videopacplus': + coreSettings.save('o2em_bios', '"g7400.bin"') + else: + coreSettings.save('o2em_bios', '"o2rom.bin"') + if (system.config['core'] == 'neocd'): + if system.isOptSet('neocd_bios'): + coreSettings.save('neocd_bios', '"' + system.config['neocd_bios'] + '"') + else: + coreSettings.save('neocd_bios', '"neocd_z.rom (CDZ)"') + """ +) + +PICO8 = text( + """ + #!/bin/sh + + TARGET_DIR="/userdata/bios" + PICO_DIR="$TARGET_DIR/pico-8" + ROMS_DIR="/userdata/roms/pico8" + TEMP_INSTALL_DIR="$ROMS_DIR/pico8-install" + PICO_8_64_FILE="$TEMP_INSTALL_DIR/pico-8/pico8_64" + PICO_8_DAT_FILE="$TEMP_INSTALL_DIR/pico-8/pico8.dat" + PICO_8_ZIPFILE=$(ls -d /userdata/roms/pico8/pico-8*raspi*.zip 2>/dev/null | head -1) + + if [ -z $PICO_8_ZIPFILE ] || [ ! -f "$PICO_8_ZIPFILE" ]; then + echo "Please add the Raspberry Pi version of Pico-8 to roms/pico8 and try again." + exit 2 + fi + + mv $PICO_8_64_FILE "$PICO_DIR/pico8" + mv $PICO_8_DAT_FILE "$PICO_DIR/" + """ +) + +YABASANSHIRO = text( + """ + from ...batoceraPaths import BIOS + + YABA_BIOS: Final = BIOS / "saturn_bios.bin" + + class YabasanshiroGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + commandArray = ["yabasanshiro", "-i", rom] + + if not (system.isOptSet('yaba_bios_hle') and system.config['yaba_bios_hle'] == '1'): + if YABA_BIOS.exists(): + commandArray[1:1] = ["-b", str(YABA_BIOS)] + return Command.Command(array=commandArray) + """ +) + +RYUJINX = text( + """ + from ...batoceraPaths import BIOS, CONFIGS + + ryujinxConf: Final = CONFIGS / "Ryujinx" + ryujinxKeys: Final = BIOS / "switch" / "prod.keys" + + class RyujinxGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + if ryujinxKeys.exists(): + shutil.copyfile(ryujinxKeys, ryujinxConf / "system" / "prod.keys") + """ +) + +DOLPHIN_PATHS = text( + """ + from ...batoceraPaths import BIOS + + DOLPHIN_BIOS: Final = BIOS / "GC" + """ +) + +DOLPHIN = text( + """ + from .dolphinPaths import DOLPHIN_BIOS + + class DolphinGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + if system.isOptSet("dolphin_SkipIPL") and system.getOptBoolean("dolphin_SkipIPL"): + ipl_regions = ["USA", "EUR", "JAP"] + if any((DOLPHIN_BIOS / region / "IPL.bin").exists() for region in ipl_regions): + dolphinSettings.set("Core", "SkipIPL", "False") + else: + dolphinSettings.set("Core", "SkipIPL", "True") + """ +) + +SCUMMVM = text( + """ + from ...batoceraPaths import BIOS + + scummExtra: Final = BIOS / "scummvm" / "extra" + + class ScummVMGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + mkdir_if_not_exists(scummExtra) + commandArray = ["/usr/bin/scummvm", "-f"] + commandArray.extend( + [f"--joystick={id}", + f"--extrapath={scummExtra}", + f"{romName}"] + ) + """ +) + +OPENMSX = text( + """ + class OpenmsxGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + with settings_tcl.open("a") as file: + file.write("filepool add -path /userdata/bios/Machines -types system_rom -position 1\\n") + """ +) + +LIBRETRO_CONFIG = text( + """ + def writeLibretroConfigToFile(retroarchConfig, system, rom): + if (system.config['core'] == 'puae') or (system.config['core'] == 'uae4arm'): + retroarchConfig['system_directory'] = '"/userdata/bios/amiga/"' + if system.config['core'] in [ 'mame', 'mess' ]: + libretroMAMEConfig.generateMAMEConfigs(controllers, system, rom, guns) + """ +) + +MAME_CONFIG = text( + """ + from ...batoceraPaths import BIOS + + def generateMAMEConfigs(playersControllers, system, rom, guns): + if system.name in [ 'mame', 'neogeo' ]: + commandLine += [ '-rompath', f'"{rom.parent};/userdata/bios/mame/;/userdata/bios/"' ] + if not system.isOptSet("sharemameart"): + artPath = f"/usr/bin/mame/artwork/;{BIOS / 'mame' / 'artwork'}" + else: + artPath = f"/usr/bin/mame/artwork/" + commandLine += [ '-artpath', artPath ] + mkdir_if_not_exists(BIOS / "mame" / "samples") + """ +) + +# hatariGenerator.py in KNULLI's older form: the language and model read +# with subscripts, the name built bare and joined to BIOS on the command line. +HATARI = text( + """ + from ...batoceraPaths import BIOS + + class HatariGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + model_mapping = {"520st_auto": { "machine": "st", "tos": "auto" }} + commandArray = ["hatari", "--fullscreen"] + machine = "st" + tosversion = "auto" + if system.isOptSet("model") and system.config["model"] in model_mapping: + machine = model_mapping[system.config["model"]]["machine"] + tosversion = model_mapping[system.config["model"]]["tos"] + toslang = "us" + if system.isOptSet("language"): + toslang = system.config["language"] + commandArray += ["--machine", machine] + tos = HatariGenerator.findBestTos(BIOS, machine, tosversion, toslang) + commandArray += [ "--tos", BIOS / tos] + + @staticmethod + def findBestTos(biosdir, machine, tos_version, language) -> str: + all_languages = ["us", "uk", ""] + all_machines_bios = { + "st": ["etos256", "104"], + } + if machine in all_machines_bios: + for v_tos_version in l_tos: + for v_language in l_lang: + if "etos" in v_tos_version: + biosversion = v_tos_version + else: + biosversion = f"tos{v_tos_version}" + filename = f"{biosversion}{v_language}.img" + if os.path.exists(f"{biosdir}/{filename}"): + return filename + raise Exception(f"no bios found for machine {machine}") + """ +) + +MELONDS = text( + """ + from ...batoceraPaths import BIOS + + class MelonDSGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + base_config = { + "DS": { + "FirmwarePath": str(BIOS / "firmware.bin"), + "BIOS7Path": str(BIOS / "bios7.bin"), + }, + "Emu": { + "ExternalBIOSEnable": True + }, + } + if system.isOptSet("melonds_console"): + base_config["Emu"]["ConsoleType"] = int(system.config["melonds_console"]) + else: + base_config["Emu"]["ConsoleType"] = 0 + """ +) + + +def tree(**overrides: str) -> dict[str, str]: + files = { + f"{kl.CONFIGGEN}/batoceraPaths.py": PATHS, + f"{G}/importer.py": IMPORTER, + f"{G}/lexaloffle/lexaloffleGenerator.py": LEXALOFFLE, + kl.LIBRETRO_OPTIONS: OPTIONS, + kl.PICO8_INSTALLER: PICO8, + f"{G}/yabasanshiro/yabasanshiroGenerator.py": YABASANSHIRO, + f"{G}/ryujinx/ryujinxGenerator.py": RYUJINX, + f"{G}/dolphin/dolphinPaths.py": DOLPHIN_PATHS, + f"{G}/dolphin/dolphinGenerator.py": DOLPHIN, + f"{G}/scummvm/scummvmGenerator.py": SCUMMVM, + f"{G}/openmsx/openmsxGenerator.py": OPENMSX, + f"{G}/libretro/libretroConfig.py": LIBRETRO_CONFIG, + f"{G}/libretro/libretroMAMEConfig.py": MAME_CONFIG, + f"{G}/hatari/hatariGenerator.py": HATARI, + f"{G}/melonds/melondsGenerator.py": MELONDS, + } + files.update(overrides) + return files + + +def configgen(**overrides: str) -> kl.Configgen: + files = tree(**overrides) + return kl.configgen(files.__getitem__) + + +def sites(path: str, kind: str, **fields: object) -> list[dict]: + spec = kl.GenericSites(path, kind, **fields) # type: ignore[arg-type] + return [launcher.entry() for launcher in kl.bios_sites(configgen(), spec)] + + +class SharedExtractors(unittest.TestCase): + """Batocera's extractors read KNULLI's older generator forms.""" + + def test_hatari_with_a_bare_name_joined_to_bios(self): + (st,) = hatari(configgen()) + self.assertEqual(st.names, ["etos256{L}.img", "tos104{L}.img"]) + self.assertEqual(st.expands, {"L": ["us", "uk", ""]}) + self.assertEqual(st.passes_to, "--machine st --tos ") + self.assertEqual(st.fails_when_missing, "no bios found for machine st") + self.assertEqual(st.depends_on_option, "model, language") + self.assertTrue(st.source_ref.startswith(f"{G}/hatari/hatariGenerator.py:")) + + def test_hatari_reads_the_knulli_package(self): + (st,) = hatari(configgen()) + self.assertNotIn("package/batocera", st.source_ref) + + def test_hatari_name_built_another_way_raises(self): + broken = HATARI.replace('filename = f"{biosversion}{v_language}.img"', "filename = x") + with self.assertRaises(LauncherShapeError): + hatari(configgen(**{f"{G}/hatari/hatariGenerator.py": broken})) + + def test_melonds_console_read_by_subscript(self): + (ds,) = melonds(configgen()) + self.assertEqual(ds.names, ["firmware.bin", "bios7.bin"]) + self.assertEqual(ds.depends_on_option, "melonds_console") + self.assertEqual(ds.source_ref, f"{G}/melonds/melondsGenerator.py:6-12, 15") + + def test_a_configgen_without_the_plain_map_raises(self): + with self.assertRaises(LauncherShapeError): + configgen(**{f"{G}/importer.py": "X = 1\n"}) + + +class Lexaloffle(unittest.TestCase): + def test_each_system_runs_its_binary(self): + pico8, vox = kl.lexaloffle(configgen()) + self.assertEqual((pico8.slot, pico8.names), ("pico8", ["pico-8/pico8"])) + self.assertEqual((vox.slot, vox.names), ("voxatron", ["voxatron/vox"])) + self.assertEqual(pico8.search_dir, "/userdata/bios") + self.assertEqual(pico8.passes_to, "exec") + + def test_the_two_checks_are_the_failure(self): + pico8, _ = kl.lexaloffle(configgen()) + self.assertEqual( + pico8.fails_when_missing, + "Lexaloffle official binary not found at /userdata/bios/pico-8/pico8, " + "File /userdata/bios/pico-8/pico8 is not set as executable", + ) + + def test_the_lines_cited(self): + pico8, vox = kl.lexaloffle(configgen()) + path = f"{G}/lexaloffle/lexaloffleGenerator.py" + self.assertEqual(pico8.source_ref, f"{path}:3, 8-9, 14-20") + self.assertEqual(vox.source_ref, f"{path}:4, 10-11, 14-20") + + def test_a_raise_in_place_of_the_log_raises(self): + broken = LEXALOFFLE.replace( + 'eslog.error(f"Lexaloffle official binary not found at {BIN_PATH}")\n return -1', + 'raise BatoceraException(f"Lexaloffle official binary not found at {BIN_PATH}")', + ) + with self.assertRaises(LauncherShapeError): + kl.lexaloffle(configgen(**{f"{G}/lexaloffle/lexaloffleGenerator.py": broken})) + + +class PicoInstaller(unittest.TestCase): + def test_the_files_moved_into_bios(self): + (record,) = kl.pico8_installer(tree().__getitem__, "/userdata/bios") + self.assertEqual(record.kind, "moves") + self.assertEqual(record.names, ["pico-8/pico8", "pico-8/pico8.dat"]) + self.assertEqual( + record.passes_to, + "moved from /userdata/roms/pico8/pico-8*raspi*.zip into /userdata/bios/pico-8", + ) + self.assertEqual( + record.fails_when_missing, + "Please add the Raspberry Pi version of Pico-8 to roms/pico8 and try again.", + ) + self.assertEqual(record.source_ref, f"{kl.PICO8_INSTALLER}:3-12, 16-17") + + def test_an_installer_that_moves_nothing_into_bios_raises(self): + broken = PICO8.replace('"$PICO_DIR/', '"/tmp/') + with self.assertRaises(LauncherShapeError): + kl.pico8_installer(tree(**{kl.PICO8_INSTALLER: broken}).__getitem__, "/userdata/bios") + + +class CoreFileOptions(unittest.TestCase): + def test_option_then_literal_defaults(self): + o2em, neocd = kl.core_file_options(configgen()) + self.assertEqual(o2em.emulator, "o2em") + self.assertEqual(o2em.names, ["g7400.bin", "o2rom.bin"]) + self.assertEqual(o2em.depends_on_option, "o2em_bios (default by system.name == 'videopacplus')") + self.assertEqual(o2em.source_ref, f"{kl.LIBRETRO_OPTIONS}:2, 4-9") + self.assertEqual((neocd.names, neocd.depends_on_option), (["neocd_z.rom (CDZ)"], "neocd_bios")) + + def test_a_default_that_is_no_literal_raises(self): + broken = OPTIONS.replace("'\"o2rom.bin\"'", "default_bios") + with self.assertRaises(LauncherShapeError): + kl.core_file_options(configgen(**{kl.LIBRETRO_OPTIONS: broken})) + + +class BiosSites(unittest.TestCase): + def test_a_flag_on_the_command_line_under_a_presence_test(self): + (record,) = sites("yabasanshiro/yabasanshiroGenerator.py", "fixed_name") + self.assertEqual(record["names"], ["saturn_bios.bin"]) + self.assertEqual(record["passes_to"], "command line -b") + self.assertEqual( + record["depends_on_option"], + "not (system.isOptSet('yaba_bios_hle') and system.config['yaba_bios_hle'] == '1')", + ) + self.assertEqual(record["source_ref"], f"{G}/yabasanshiro/yabasanshiroGenerator.py:9-11") + + def test_a_copy_out_of_bios(self): + (record,) = sites("ryujinx/ryujinxGenerator.py", "fixed_name") + self.assertEqual(record["kind"], "copies") + self.assertEqual(record["names"], ["switch/prod.keys"]) + self.assertEqual(record["passes_to"], "copied to /userdata/system/configs/Ryujinx/system/prod.keys") + self.assertNotIn("depends_on_option", record) + + def test_a_presence_test_over_a_literal_list(self): + (record,) = sites("dolphin/dolphinGenerator.py", "fixed_name") + self.assertEqual(record["names"], ["GC/{region}/IPL.bin"]) + self.assertEqual(record["expands"], {"region": ["USA", "EUR", "JAP"]}) + self.assertEqual(record["passes_to"], "presence sets [Core] SkipIPL = False") + self.assertEqual(record["source_ref"], f"{G}/dolphin/dolphinGenerator.py:5, 7-8") + + def test_a_flag_written_into_the_argument(self): + (record,) = sites("scummvm/scummvmGenerator.py", "passes_path") + self.assertEqual(record["search_dir"], "/userdata/bios/scummvm/extra") + self.assertEqual(record["passes_to"], "command line --extrapath") + + def test_a_line_written_to_a_config(self): + (record,) = sites("openmsx/openmsxGenerator.py", "passes_path") + self.assertEqual(record["search_dir"], "/userdata/bios/Machines") + self.assertTrue(record["passes_to"].startswith("written: filepool add -path")) + + def test_a_core_test_names_the_emulator(self): + (record,) = sites("libretro/libretroConfig.py", "passes_path", build="libretro") + self.assertEqual(record["emulator"], "puae, uae4arm") + self.assertEqual(record["passes_to"], "system_directory") + + def test_a_helper_takes_the_cores_of_its_caller(self): + records = sites( + "libretro/libretroMAMEConfig.py", "passes_path", build="libretro", + caller=("libretro/libretroConfig.py", "generateMAMEConfigs"), + ) + self.assertEqual({r["emulator"] for r in records}, {"mame, mess"}) + # The else branch binds an art path outside /userdata/bios: no record. + rompath, art = records + self.assertEqual(rompath["search_dir"], ";/userdata/bios/mame/;/userdata/bios/") + self.assertEqual(rompath["passes_to"], "command line -rompath") + self.assertEqual(art["search_dir"], "/usr/bin/mame/artwork/;/userdata/bios/mame/artwork") + self.assertEqual(art["depends_on_option"], "not system.isOptSet('sharemameart')") + self.assertEqual(art["source_ref"], f"{G}/libretro/libretroMAMEConfig.py:6-7, 10") + + def test_a_generator_without_a_bios_path_raises(self): + files = tree(**{f"{G}/openmsx/openmsxGenerator.py": "def generate():\n return 1\n"}) + spec = kl.GenericSites("openmsx/openmsxGenerator.py", "passes_path") + with self.assertRaises(LauncherShapeError): + kl.bios_sites(kl.configgen(files.__getitem__), spec) + + +class PlatformFile(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.config = yaml.safe_load((REPO_ROOT / "platforms" / "knulli.yml").read_text()) + + def test_the_layer_is_written(self): + self.assertTrue(self.config["launchers"]) + + def test_every_reference_is_in_knulli_code(self): + for record in self.config["launchers"]: + self.assertTrue( + record["source_ref"].startswith((f"{kl.CONFIGGEN}/", kl.PICO8_INSTALLER)), record + ) + + def test_every_emulator_is_one_knulli_names(self): + cores = {*self.config["cores"], *self.config.get("standalone_cores", [])} + named = {n for record in self.config["launchers"] for n in record["emulator"].split(", ")} + # RetroArch itself, and the MAME core names configgen tests that + # KNULLI's es_systems.yml never offers. + self.assertEqual(sorted(named - cores), ["mamevirtual", "mess", "retroarch"]) + + +if __name__ == "__main__": + unittest.main() From ffad06487cde8f136dfb2534f00776cd0fddce5b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:47:23 +0100 Subject: [PATCH 3/7] feat: read the amberelec launcher layer amberelec_launchers reads runemu.py with ast and, at the commit the scraper pins, the scripts its standalone table runs and the configs their packages install: RetroArch and retrorun directories, the MAME -rp path, atari800.cfg, Hatari's default TOS, amiberry's kickstarts link and defaults, yabasanshiro, PICO-8, piemu, scummvm, freej2me and easyrpg. 17 records. The freej2me-lr.jar and pico8_64 notes now name the file in a sentence about the emulator, as test_launcher_aliases asks once a launcher records it. --- emulators/freej2me.yml | 2 +- emulators/lexaloffle.yml | 2 +- platforms/amberelec.yml | 181 +++++++++++ scripts/scraper/amberelec_launchers.py | 429 +++++++++++++++++++++++++ scripts/scraper/amberelec_scraper.py | 7 + tests/test_launchers_amberelec.py | 345 ++++++++++++++++++++ 6 files changed, 964 insertions(+), 2 deletions(-) create mode 100644 scripts/scraper/amberelec_launchers.py create mode 100644 tests/test_launchers_amberelec.py diff --git a/emulators/freej2me.yml b/emulators/freej2me.yml index 45a5bc303..85c6af334 100644 --- a/emulators/freej2me.yml +++ b/emulators/freej2me.yml @@ -31,4 +31,4 @@ files: required: true installed_on: [retrobat, rocknix] source_ref: "src/libretro/freej2me_libretro.c:487-490 (GET_SYSTEM_DIRECTORY + fill_pathname_join + java -jar)" - note: "Not a BIOS dump. Java emulator loaded by the native core at startup via execvp('java', '-jar', path). Built from source via 'ant'. Must be in the system directory root. ROCKNIX builds the core and the jar from freej2me-plus 1.52, which keeps the freej2me-lr.jar name, and copies the jar into its BIOS folder before each launch when it is missing." + note: "Not a BIOS dump. Java emulator the native core joins as freej2me-lr.jar to the system directory and starts via execvp('java', '-jar', path). Built from source via 'ant'. Must be in the system directory root. ROCKNIX builds the core and the jar from freej2me-plus 1.52, which keeps the freej2me-lr.jar name, and copies the jar into its BIOS folder before each launch when it is missing." diff --git a/emulators/lexaloffle.yml b/emulators/lexaloffle.yml index 1fa22616a..28f058674 100644 --- a/emulators/lexaloffle.yml +++ b/emulators/lexaloffle.yml @@ -130,7 +130,7 @@ files: md5: 70ab37eab4e7d162d36d9b2d91b84867 sha1: 4ed23b6578a2d93d67aa856791dc70e95d68a3f4 description: "PICO-8 0.2.7 aarch64 executable, system libSDL2" - note: "Only in pico-8_0.2.7_raspi.zip, the build for 64-bit Raspberry Pi OS, linked against the system SDL2. AmberELEC and ROCKNIX run it from roms/pico-8/, not from bios/. AmberELEC stops with a message when it is missing. ROCKNIX picks it when HW_ARCH is aarch64. Recalbox renames bios/pico-8/pico8_64 to pico8 before the first run, and a missing file fails the rename." + note: "pico8_64 is only in pico-8_0.2.7_raspi.zip, the build for 64-bit Raspberry Pi OS, linked against the system SDL2. AmberELEC and ROCKNIX run it from roms/pico-8/, not from bios/. AmberELEC stops with a message when it is missing. ROCKNIX picks it when HW_ARCH is aarch64. Recalbox renames bios/pico-8/pico8_64 to pico8 before the first run, and a missing file fails the rename." source_ref: "pico-8 0.2.7 raspi readme_raspi.txt:3-11, amberelec packages/amberelec/sources/pico-8.sh:51-54, amberelec packages/amberelec/sources/scripts/runemu.py:123, rocknix projects/ROCKNIX/packages/emulators/standalone/pico-8/scripts/start_pico8.sh:8-17, 35-40, 46-47, recalbox projects/configgen/configgen/generators/pico8/pico8Generator.py:14-19, dArkOS pico8/pico8.sh:47-60, 128-135" - name: pico8_dyn diff --git a/platforms/amberelec.yml b/platforms/amberelec.yml index c6e472540..a4085d839 100644 --- a/platforms/amberelec.yml +++ b/platforms/amberelec.yml @@ -3172,3 +3172,184 @@ systems: cores: - fuse - ep128emu_core +launchers: +- emulator: retroarch + build: libretro + kind: passes_path + names: [] + search_dir: ~/roms/bios + passes_to: retroarch.cfg system_directory + fails_when_missing: false + source_ref: "packages/games/tools/retroarch/sources/retroarch.cfg:611" +- emulator: retrorun + build: libretro + kind: passes_path + names: [] + search_dir: /roms/bios + passes_to: command line -d + fails_when_missing: false + source_ref: "packages/games/tools/retrorun/retrorun.sh:583" +- emulator: mame + build: libretro + kind: passes_path + names: [] + search_dir: /roms/bios/mame;/roms/bios;/roms/arcade;/roms/mame + passes_to: .cmd -rp + fails_when_missing: false + depends_on_option: self.emulator == 'mame' + source_ref: "packages/amberelec/sources/scripts/runemu.py:233, 252, 255" +- emulator: atari800 + build: libretro + kind: fixed_name + names: + - ATARIOSA.ROM + - ATARIXL.ROM + - 5200.rom + - 5200a.rom + - BASIC_Rev_A.rom + - BASIC_Rev_B.rom + - ATARIBAS.ROM + - ATARIOSB.ROM + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: atari800.cfg ROM_OS_A_PAL, ROM_OS_BB01R2, ROM_5200, ROM_5200_A, ROM_BASIC_A, + ROM_BASIC_B, ROM_BASIC_C, ROM_400/800_CUSTOM + fails_when_missing: false + source_ref: "packages/games/libretro/atari800/patches/atari800-01-add-emuelec-platform.patch:27,\ + \ packages/amberelec/config/distribution/configs/atari800.cfg:3, 8, 16-22" +- emulator: scummvm + build: libretro + kind: passes_path + names: [] + search_dir: /roms/bios/scummvm/theme + passes_to: scummvm.ini themepath + fails_when_missing: false + depends_on_option: scummvm.sh libretro + source_ref: "packages/games/emulators/scummvmsa/scummvm.sh:34, 42" +- emulator: scummvm + build: libretro + kind: passes_path + names: [] + search_dir: /roms/bios/scummvm/extra + passes_to: scummvm.ini extrapath + fails_when_missing: false + depends_on_option: scummvm.sh libretro + source_ref: "packages/games/emulators/scummvmsa/scummvm.sh:34, 43" +- emulator: freej2me + build: libretro + kind: copies + names: + - freej2me-lr.jar + case_insensitive: false + search_dir: /usr/config/distribution/freej2me + passes_to: /storage/roms/bios + fails_when_missing: false + source_ref: "packages/games/libretro/freej2me/freej2me.sh:11-12, 44" +- emulator: freej2me-plus + build: libretro + kind: copies + names: + - freej2me-plus-lr.jar + case_insensitive: false + search_dir: /usr/config/distribution/freej2me + passes_to: /storage/roms/bios + fails_when_missing: false + source_ref: "packages/games/libretro/freej2me-plus/freej2me-plus.sh:11-12, 44" +- emulator: easyrpg + build: libretro + kind: copies + names: [] + search_dir: "https://dl.degica.com/rpgmakerweb/run-time-packages/rpg2000_rtp_installer.exe" + passes_to: /storage/roms/bios/rtp/2000 + fails_when_missing: false + source_ref: "packages/games/libretro/easyrpg/easyrpg.sh:23, 42, 46, 60" +- emulator: easyrpg + build: libretro + kind: copies + names: [] + search_dir: "https://dl.degica.com/rpgmakerweb/run-time-packages/rpg2003_rtp_installer.zip" + passes_to: /storage/roms/bios/rtp/2003 + fails_when_missing: false + source_ref: "packages/games/libretro/easyrpg/easyrpg.sh:23, 42, 46, 61" +- emulator: AMIBERRY + build: standalone + kind: links + names: [] + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: /usr/config/amiberry/kickstarts is a symlink to it + fails_when_missing: false + source_ref: "packages/games/emulators/amiberry/package.mk:35" +- emulator: AMIBERRY + build: standalone + kind: fixed_name + slot: A1200 + names: + - kick40068.A1200.rom + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: "AmigaA1200-default.uae kickstart_rom_file, $(FILE_PATH) = /storage/.config/amiberry/kickstarts/" + fails_when_missing: false + depends_on_option: "[ `echo $1 | egrep 'AGA|CD32' | wc -l` -eq 1 -o `echo \"$AMIBERRY_TMP_DIR\"\ + /* | egrep 'AGA|CD32' | wc -l` -eq 1 ]" + source_ref: "packages/amberelec/sources/scripts/runemu.py:111, packages/games/emulators/amiberry/scripts/amiberry.sh:57-59,\ + \ packages/games/emulators/amiberry/config/conf/AmigaA1200-default.uae:35, 85" +- emulator: AMIBERRY + build: standalone + kind: fixed_name + slot: A500 + names: + - kick34005.A500.rom + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: "AmigaA500-default.uae kickstart_rom_file, $(FILE_PATH) = /storage/.config/amiberry/kickstarts/" + fails_when_missing: false + depends_on_option: "not ([ `echo $1 | egrep 'AGA|CD32' | wc -l` -eq 1 -o `echo \"\ + $AMIBERRY_TMP_DIR\"/* | egrep 'AGA|CD32' | wc -l` -eq 1 ])" + source_ref: "packages/amberelec/sources/scripts/runemu.py:111, packages/games/emulators/amiberry/scripts/amiberry.sh:60-62,\ + \ packages/games/emulators/amiberry/config/conf/AmigaA500-default.uae:35, 85" +- emulator: HATARISA + build: standalone + kind: fixed_name + names: + - tos.img + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: "Atari-ST-default.cfg [ROM] szTosImageFileName" + fails_when_missing: false + source_ref: "packages/amberelec/sources/scripts/runemu.py:113, packages/games/emulators/hatarisa/scripts/hatari.sh:36,\ + \ 66, packages/games/emulators/hatarisa/config/conf/Atari-ST-default.cfg:200" +- emulator: pico8 + build: standalone + kind: fixed_name + names: + - pico8_64 + - pico8.dat + case_insensitive: false + search_dir: /storage/roms/pico-8 + passes_to: exec + fails_when_missing: Missing Pico-8 binaries! + source_ref: "packages/amberelec/sources/scripts/runemu.py:123, packages/amberelec/sources/pico-8.sh:51-54" +- emulator: piemu + build: standalone + kind: fixed_name + names: + - all.bin + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: mkpfi argument + fails_when_missing: false + source_ref: "packages/amberelec/sources/scripts/runemu.py:124, packages/games/emulators/piemu/bin/piemu.sh:8-10,\ + \ 33, 37, 50" +- emulator: yabasanshiroSA + build: standalone + kind: fixed_name + names: + - saturn_bios.bin + case_insensitive: false + search_dir: /storage/roms/bios + passes_to: command line -b + fails_when_missing: false + depends_on_option: use_hlebios + source_ref: "packages/amberelec/sources/scripts/runemu.py:127, packages/games/emulators/yabasanshiroSA/yabasanshiro.sh:36,\ + \ 63-65, 78" diff --git a/scripts/scraper/amberelec_launchers.py b/scripts/scraper/amberelec_launchers.py new file mode 100644 index 000000000..725c8cced --- /dev/null +++ b/scripts/scraper/amberelec_launchers.py @@ -0,0 +1,429 @@ +"""AmberELEC's launcher layer, read from runemu.py and its emulator scripts. + +EmulationStation hands every launch to runemu.py. A libretro core runs in +RetroArch or retrorun with the system directory their configs name, MAME +machines get a .cmd file with a -rp search path, and every other emulator +runs the script its standalone_emulators table names (/usr/bin/.sh, +installed from the package). Those scripts, and the configs their packages +install, say which file under /storage/roms/bios is opened: yabasanshiro.sh +tests saturn_bios.bin and passes it with -b, Hatari's default config names +tos.img, the atari800 core is built to read a config naming its OS ROMs. + +runemu.py is read with ast, the scripts with shell_launchers' readers. An +extractor whose construct is gone raises instead of returning nothing. +""" + +from __future__ import annotations + +import ast +import posixpath +import re +from collections.abc import Callable +from typing import Any + +try: + from .configgen_launchers import Launcher, Source + from .shell_launchers import ( + LauncherPatternMissing, + Script, + describe_keys, + settings, + source_ref, + ) +except ImportError: + from configgen_launchers import Launcher, Source # type: ignore[no-redef] + from shell_launchers import ( # type: ignore[no-redef] + LauncherPatternMissing, + Script, + describe_keys, + settings, + source_ref, + ) + +BIOS = "/storage/roms/bios" +RUNEMU = "packages/amberelec/sources/scripts/runemu.py" +RETROARCH_CFG = "packages/games/tools/retroarch/sources/retroarch.cfg" +RETRORUN = "packages/games/tools/retrorun/retrorun.sh" +HATARI_CONFIG = "packages/games/emulators/hatarisa/config/conf/Atari-ST-default.cfg" +ATARI800_CONFIG = "packages/amberelec/config/distribution/configs/atari800.cfg" +ATARI800_PATCH = "packages/games/libretro/atari800/patches/atari800-01-add-emuelec-platform.patch" +AMIBERRY_PACKAGE = "packages/games/emulators/amiberry/package.mk" +AMIBERRY_CONFIGS = "packages/games/emulators/amiberry/config/conf" +FREEJ2ME = ( + ("freej2me", "packages/games/libretro/freej2me/freej2me.sh"), + ("freej2me-plus", "packages/games/libretro/freej2me-plus/freej2me-plus.sh"), +) +EASYRPG = "packages/games/libretro/easyrpg/easyrpg.sh" + +# The script each standalone_emulators entry runs, as the package installs it +# to /usr/bin. scripts() checks each against runemu's table. +SCRIPTS = { + "AMIBERRY": "packages/games/emulators/amiberry/scripts/amiberry.sh", + "HATARISA": "packages/games/emulators/hatarisa/scripts/hatari.sh", + "SCUMMVMSA": "packages/games/emulators/scummvmsa/scummvm.sh", + "pico8": "packages/amberelec/sources/pico-8.sh", + "piemu": "packages/games/emulators/piemu/bin/piemu.sh", + "yabasanshiroSA": "packages/games/emulators/yabasanshiroSA/yabasanshiro.sh", +} + +Read = Callable[[str], str] + + +class Runemu: + """runemu.py, parsed: the standalone table and the RetroArch command.""" + + def __init__(self, read: Read) -> None: + self.src = Source(RUNEMU, read(RUNEMU)) + node = self.src.assignment("standalone_emulators") + if not isinstance(node.value, ast.Dict): + self.src.fail("standalone_emulators is not a dict display") + self.table: dict[str, tuple[str, int]] = {} + for key, value in zip(node.value.keys, node.value.values): + if not ( + isinstance(key, ast.Constant) + and isinstance(value, ast.Call) + and len(value.args) >= 2 + and isinstance(value.args[1], ast.List) + ): + self.src.fail(f"entry at line {getattr(key, 'lineno', '?')} is not StandaloneEmulator(name, [args])") + args = [a.value for a in value.args[1].elts if isinstance(a, ast.Constant)] + script = next((a for a in args if str(a).endswith(".sh")), None) + if script is None: + self.src.fail(f"{ast.unparse(key)} runs no .sh script") + self.table[str(key.value)] = (str(script), value.lineno) + + def script(self, emulator: str) -> tuple[str, int]: + """The repository script runemu runs for emulator, and the table line.""" + if emulator not in self.table: + self.src.fail(f"standalone_emulators has no {emulator}") + installed, line = self.table[emulator] + path = SCRIPTS[emulator] + if posixpath.basename(installed) != posixpath.basename(path): + self.src.fail(f"{emulator} runs {installed}, not {posixpath.basename(path)}") + return path, line + + +def retroarch(read: Read) -> list[Launcher]: + """system_directory of the retroarch.cfg the retroarch package installs.""" + found = [s for s in settings(read(RETROARCH_CFG)) if s.key == "system_directory"] + if len(found) != 1: + raise LauncherPatternMissing(RETROARCH_CFG, "one system_directory") + return [Launcher( + emulator="retroarch", build="libretro", kind="passes_path", names=[], + search_dir=found[0].value, passes_to="retroarch.cfg system_directory", + fails_when_missing=False, source_ref=source_ref([(RETROARCH_CFG, [found[0].line])]), + )] + + +def retrorun(read: Read) -> list[Launcher]: + """The -d directory retrorun.sh passes to retrorun.""" + script = Script(RETRORUN, read(RETRORUN)) + line, found = script.find(r"^\s*/usr/bin/retrorun\s.*\s-d\s+(\S+)", "retrorun -d") + return [Launcher( + emulator="retrorun", build="libretro", kind="passes_path", names=[], + search_dir=found.group(1).strip('"'), passes_to="command line -d", + fails_when_missing=False, source_ref=source_ref([(RETRORUN, [line])]), + )] + + +def mame(runemu: Runemu) -> list[Launcher]: + """The -rp search path of the .cmd file runemu writes for a MAME machine.""" + src = runemu.src + commands = [ + node for node in ast.walk(src.function("get_retroarch_command")) + if isinstance(node, ast.JoinedStr) + and any(isinstance(p, ast.Constant) and " -rp " in str(p.value) for p in node.values) + ] + if not commands: + src.fail("get_retroarch_command writes no -rp search path") + paths = [] + for node in commands: + text = "".join(str(p.value) for p in node.values if isinstance(p, ast.Constant)) + paths.append(text.split(" -rp ", 1)[1].split()[0]) + if len(set(paths)) != 1: + src.fail(f"the MAME commands differ in their search path: {sorted(set(paths))}") + guard = next( + node for node in ast.walk(src.function("get_retroarch_command")) + if isinstance(node, ast.If) and ast.unparse(node.test) == "self.emulator == 'mame'" + ) + return [Launcher( + emulator="mame", build="libretro", kind="passes_path", names=[], + search_dir=paths[0], passes_to=".cmd -rp", fails_when_missing=False, + depends_on_option=ast.unparse(guard.test), + source_ref=src.ref([ + src.span(guard.test), *((n.lineno, n.end_lineno or n.lineno) for n in commands), + ]), + )] + + +def _bios_files(path: str, text: str) -> tuple[list[tuple[tuple[str, str], str]], set[int]]: + """((section, key), name under BIOS) for each config key whose value is a BIOS file.""" + prefix = BIOS + "/" + found = [s for s in settings(text) if s.value.startswith(prefix)] + if not found: + raise LauncherPatternMissing(path, f"key naming a file under {BIOS}") + return [((s.section, s.key), s.value[len(prefix):]) for s in found], {s.line for s in found} + + +def hatari(read: Read, runemu: Runemu) -> list[Launcher]: + """szTosImageFileName of the default config hatari.sh copies and passes.""" + path, table_line = runemu.script("HATARISA") + script = Script(path, read(path)) + name = posixpath.basename(HATARI_CONFIG) + copied, _ = script.find(rf"^\s*cp\s+\S*/{re.escape(name)}\s", f"cp of {name}") + passed, _ = script.find(r"\shatari\s+--configfile\s", "hatari --configfile") + files, lines = _bios_files(HATARI_CONFIG, read(HATARI_CONFIG)) + return [Launcher( + emulator="HATARISA", build="standalone", kind="fixed_name", + names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS, + passes_to=describe_keys(name, tuple(k for k, _n in files)), + fails_when_missing=False, + source_ref=source_ref([ + (RUNEMU, [table_line]), (path, [copied, passed]), (HATARI_CONFIG, lines), + ]), + )] + + +def atari800(read: Read) -> list[Launcher]: + """The ROM keys of the config the atari800 core is built to read.""" + patch = Script(ATARI800_PATCH, read(ATARI800_PATCH)) + line, found = patch.find(r'DEFAULT_CFG_NAME="\\"([^"\\]+)\\""', "DEFAULT_CFG_NAME") + built = found.group(1) + if not built.endswith("/" + ATARI800_CONFIG.split("/config/", 1)[1]): + raise LauncherPatternMissing(ATARI800_PATCH, f"DEFAULT_CFG_NAME naming {ATARI800_CONFIG}") + files, lines = _bios_files(ATARI800_CONFIG, read(ATARI800_CONFIG)) + return [Launcher( + emulator="atari800", build="libretro", kind="fixed_name", + names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS, + passes_to=describe_keys(posixpath.basename(ATARI800_CONFIG), tuple(k for k, _n in files)), + fails_when_missing=False, + source_ref=source_ref([(ATARI800_PATCH, [line]), (ATARI800_CONFIG, lines)]), + )] + + +def yabasanshiro(read: Read, runemu: Runemu) -> list[Launcher]: + """`if [ -e ]; then BIOS="-b "`, skipped when HLE is chosen.""" + path, table_line = runemu.script("yabasanshiroSA") + script = Script(path, read(path)) + tested, test = script.find(rf'if \[ -e "({re.escape(BIOS)}/[^"]+)" \]', "BIOS presence test") + flag_line, flag = script.find( + rf'BIOS="(-\w+) {re.escape(test.group(1))}"', "BIOS flag", tested + ) + hle_line, hle = script.find(r'get_setting "(\w*hle\w*)"', "HLE setting") + guard_line, _ = script.find(r'if \[ "\$\{HLEBIOS\}" != 1 \]', "HLE test") + used_line, _ = script.find(r"\$\{BIOS\}", "BIOS on the command line", flag_line + 1) + return [Launcher( + emulator="yabasanshiroSA", build="standalone", kind="fixed_name", + names=[test.group(1)[len(BIOS) + 1:]], case_insensitive=False, search_dir=BIOS, + passes_to=f"command line {flag.group(1)}", fails_when_missing=False, + depends_on_option=hle.group(1), + source_ref=source_ref([ + (RUNEMU, [table_line]), + (path, [hle_line, guard_line, tested, flag_line, used_line]), + ]), + )] + + +def pico8(read: Read, runemu: Runemu) -> list[Launcher]: + """`if [ ! -f A ] || [ ! -f B ]; then text_viewer ...; else A ...`.""" + path, table_line = runemu.script("pico8") + script = Script(path, read(path)) + line, test = script.find(r'if \[ ! -f "([^"]+)" \] \|\| \[ ! -f "([^"]+)" \]', "binary test") + message_line, message = script.find(r'text_viewer .*-t "([^"]+)"', "missing message", line) + binary = test.group(1) + run_line, _ = script.find(rf"^\s*{re.escape(binary)}\s", f"{binary} run", message_line) + folders = {posixpath.dirname(test.group(1)), posixpath.dirname(test.group(2))} + if len(folders) != 1: + raise script.missing("both binaries in one folder") + return [Launcher( + emulator="pico8", build="standalone", kind="fixed_name", + names=[posixpath.basename(test.group(1)), posixpath.basename(test.group(2))], + case_insensitive=False, search_dir=folders.pop(), passes_to="exec", + fails_when_missing=message.group(1), + source_ref=source_ref([(RUNEMU, [table_line]), (path, [line, message_line, run_line])]), + )] + + +def piemu(read: Read, runemu: Runemu) -> list[Launcher]: + """The BIOS file piemu.sh hands to mkpfi to build the flash image.""" + path, table_line = runemu.script("piemu") + script = Script(path, read(path)) + names: list[str] = [] + lines: set[int] = set() + for number, line in script.code(): + found = re.search(r'/usr/bin/mkpfi\s+"([^"]+)"', line) + if found is None: + continue + used: set[int] = {number} + value = script.expand(found.group(1), number, used) + if value.startswith(BIOS + "/") and value[len(BIOS) + 1:] not in names: + names.append(value[len(BIOS) + 1:]) + lines |= used + if not names: + raise script.missing(f"mkpfi of a file under {BIOS}") + return [Launcher( + emulator="piemu", build="standalone", kind="fixed_name", names=names, + case_insensitive=False, search_dir=BIOS, passes_to="mkpfi argument", + fails_when_missing=False, + source_ref=source_ref([(RUNEMU, [table_line]), (path, lines)]), + )] + + +def amiberry(read: Read, runemu: Runemu) -> list[Launcher]: + """The kickstarts link of the package, and the ROM each default config names.""" + package = Script(AMIBERRY_PACKAGE, read(AMIBERRY_PACKAGE)) + links = [link for link in package.links(BIOS) if link.bios_is_target] + if len(links) != 1 or links[0].bios_side.rstrip("/") != BIOS: + raise package.missing(f"ln -s {BIOS} to the kickstarts folder") + link = links[0] + installed = re.sub(r"^\$\{INSTALL\}", "", link.link) + path, table_line = runemu.script("AMIBERRY") + script = Script(path, read(path)) + launchers = [Launcher( + emulator="AMIBERRY", build="standalone", kind="links", names=[], + case_insensitive=False, search_dir=BIOS, + passes_to=f"{installed} is a symlink to it", fails_when_missing=False, + source_ref=source_ref([(AMIBERRY_PACKAGE, link.lines)]), + )] + for number, line in script.code(): + copied = re.search(r"cp\s+\$AMIBERRY_CONFIG_DIR/(\S+\.uae)\s", line) + if copied is None: + continue + config = f"{AMIBERRY_CONFIGS}/{copied.group(1)}" + found = [ + s for s in settings(read(config)) + if s.key == "kickstart_rom_file" and s.value.startswith("$(FILE_PATH)/") + ] + rom_path = [s for s in settings(read(config)) if s.key == "amiberry.rom_path"] + if len(found) != 1 or len(rom_path) != 1: + raise LauncherPatternMissing(config, "kickstart_rom_file under $(FILE_PATH)") + if rom_path[0].value.rstrip("/").rsplit("/", 1)[-1] != installed.rstrip("/").rsplit("/", 1)[-1]: + raise LauncherPatternMissing(config, f"amiberry.rom_path naming {installed}") + condition_line = max( + n for n, text in script.code() + if n < number and re.match(r"\s*(if|else)\b", text) + ) + branch = script.lines[condition_line - 1].strip() + if branch == "else": + opened = max( + n for n, text in script.code() + if n < condition_line and re.match(r"\s*if\b", text) + ) + test = _shell_test(script, opened) + condition = f"not ({test})" + else: + test = _shell_test(script, condition_line) + condition = test + launchers.append(Launcher( + emulator="AMIBERRY", build="standalone", kind="fixed_name", + slot=copied.group(1).removesuffix("-default.uae").removeprefix("Amiga"), + names=[found[0].value.removeprefix("$(FILE_PATH)/")], case_insensitive=False, + search_dir=BIOS, + passes_to=f"{copied.group(1)} kickstart_rom_file, $(FILE_PATH) = {rom_path[0].value}", + fails_when_missing=False, + depends_on_option=condition, + source_ref=source_ref([ + (RUNEMU, [table_line]), (path, [condition_line, number]), + (config, [rom_path[0].line, found[0].line]), + ]), + )) + if len(launchers) == 1: + raise script.missing("cp of a default .uae config") + return launchers + + +def _shell_test(script: Script, line: int) -> str: + """The test of `if ; then` at a line.""" + found = re.match(r"\s*if\s+(.*?);\s*then\s*$", script.lines[line - 1]) + if found is None: + raise script.missing(f"if ...; then at line {line}") + return found.group(1) + + +def scummvm(read: Read) -> list[Launcher]: + """The paths scummvm.sh writes into the libretro core's scummvm.ini.""" + path = SCRIPTS["SCUMMVMSA"] + script = Script(path, read(path)) + branch, _ = script.find(r'^\s*"libretro"\)', "libretro branch") + launchers = [] + for number, line in script.code(): + if number < branch: + continue + written = re.search(r"s/\^/(\w*path)=((?:\\/[^\\/n]+)+)\\n/", line) + if written is None: + continue + folder = written.group(2).replace("\\/", "/") + if "/roms/bios/" not in folder + "/" or written.group(1) == "browser_lastpath": + continue + launchers.append(Launcher( + emulator="scummvm", build="libretro", kind="passes_path", names=[], + search_dir=folder, passes_to=f"scummvm.ini {written.group(1)}", + fails_when_missing=False, depends_on_option="scummvm.sh libretro", + source_ref=source_ref([(path, [branch, number])]), + )) + if not launchers: + raise script.missing("a scummvm.ini path written under /roms/bios") + return launchers + + +def freej2me(read: Read) -> list[Launcher]: + """`if [ ! -f / ]; then cp `.""" + launchers = [] + for core, path in FREEJ2ME: + script = Script(path, read(path)) + tested, test = script.find( + rf'if \[ ! -f "{re.escape(BIOS)}/([^"]+)" \]', "jar presence test" + ) + jar = test.group(1) + lines = [tested] + source = None + for number, line in script.code(): + copied = re.match(rf"\s*cp\s+(?:-\S+\s+)*(\S+/{re.escape(jar)})\s+{re.escape(BIOS)}/?\s*$", line) + if copied: + lines.append(number) + source = copied.group(1) + if source is None: + raise script.missing(f"cp of {jar} into {BIOS}") + launchers.append(Launcher( + emulator=core, build="libretro", kind="copies", names=[jar], + case_insensitive=False, search_dir=posixpath.dirname(source), passes_to=BIOS, + fails_when_missing=False, source_ref=source_ref([(path, lines)]), + )) + return launchers + + +def easyrpg(read: Read) -> list[Launcher]: + """Each install_bios call: a runtime package downloaded and unpacked under BIOS.""" + script = Script(EASYRPG, read(EASYRPG)) + url_line, url = script.find(r'curl -Lo "\$\{BIOS\}" "([^"]+)"', "curl of the package") + launchers = [] + for number, line in script.code(): + call = re.match(r"\s*install_bios\s+(\S+)\s+(\S+)\s*$", line) + if call is None: + continue + used: set[int] = {number, url_line} + source = script.expand(url.group(1), number, used, {"BIOS": call.group(1)}) + install = script.value("INSTALLPATH", number) + if install is None or install.value.rstrip("/") != BIOS: + raise script.missing(f"INSTALLPATH {BIOS}") + used.add(install.line) + launchers.append(Launcher( + emulator="easyrpg", build="libretro", kind="copies", names=[], + search_dir=source, passes_to=f"{BIOS}/{call.group(2)}", + fails_when_missing=False, source_ref=source_ref([(EASYRPG, used)]), + )) + if not launchers: + raise script.missing("install_bios call") + return launchers + + +def extract_launchers(read: Read) -> list[dict[str, Any]]: + """Every launcher record, read through `read(repo-relative path)`.""" + runemu = Runemu(read) + launchers = [ + *retroarch(read), *retrorun(read), *mame(runemu), + *atari800(read), *scummvm(read), *freej2me(read), *easyrpg(read), + *amiberry(read, runemu), *hatari(read, runemu), *pico8(read, runemu), + *piemu(read, runemu), *yabasanshiro(read, runemu), + ] + return [launcher.entry() for launcher in launchers] diff --git a/scripts/scraper/amberelec_scraper.py b/scripts/scraper/amberelec_scraper.py index 0f729fae8..1ce5dfbe2 100644 --- a/scripts/scraper/amberelec_scraper.py +++ b/scripts/scraper/amberelec_scraper.py @@ -23,6 +23,9 @@ Verification (batocera-systems at that commit): (packages/amberelec/profile.d/99-distribution.conf:320) passes `--strictfilter`, which the script's argument switch (lines 665-687) does not handle: that call prints nothing and exits 0. + +The launcher layer is read at the same commit (amberelec_launchers): runemu.py, +the scripts its standalone table runs and the configs their packages install. """ from __future__ import annotations @@ -32,6 +35,7 @@ import re import urllib.error import urllib.request +from .amberelec_launchers import extract_launchers from .base_scraper import BaseScraper, BiosRequirement, github_headers from .batocera_scraper import ( SYSTEM_SLUG_MAP as BATOCERA_SLUGS, @@ -155,6 +159,9 @@ class Scraper(BaseScraper): "standalone_cores": standalone, "dual_mode_cores": dual, "systems": systems, + "launchers": extract_launchers( + lambda path: self._fetch_text(_RAW.format(ref=self.commit) + path) + ), } diff --git a/tests/test_launchers_amberelec.py b/tests/test_launchers_amberelec.py new file mode 100644 index 000000000..c0ba27d36 --- /dev/null +++ b/tests/test_launchers_amberelec.py @@ -0,0 +1,345 @@ +"""AmberELEC's launcher layer, read from runemu.py and its emulator scripts. + +The fixtures are cut-down copies of what each extractor reads at AmberELEC +a21975bf (prerelease-20261019_1835). Records must name the files in the +order the code writes them, cite the fixture's own lines, and a construct +that is gone must stop the scrape. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import sys +import textwrap +import unittest +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from scraper import amberelec_launchers as al # noqa: E402 +from scraper.configgen_launchers import LauncherShapeError # noqa: E402 + + +def text(block: str) -> str: + return textwrap.dedent(block).lstrip("\n") + + +RUNEMU = text( + """ + standalone_emulators = { + 'AMIBERRY': StandaloneEmulator('amiberry', ['/usr/bin/amiberry.sh', '']), + 'HATARISA': StandaloneEmulator('hatari', ['/usr/bin/hatari.sh', '']), + 'pico8': StandaloneEmulator('pico8_64', ['/usr/bin/pico-8.sh', '']), + 'piemu': StandaloneEmulator('piemu', ['/usr/bin/bash', '-l', '/usr/bin/piemu.sh', '']), + 'yabasanshiroSA': StandaloneEmulator('yabasanshiro', ['/usr/bin/yabasanshiro.sh', '']), + } + + class EmuRunner(): + def get_retroarch_command(self, shader_arg): + if self.emulator == 'mame': + if rom_path: + cmd = f'{mame_system} -{mame_slot} "{rom_path}" -rp /roms/bios/mame;/roms/bios -cfg_directory /x' + else: + cmd = f'{mame_system} -rp /roms/bios/mame;/roms/bios -cfg_directory /x' + """ +) + +YABASANSHIRO = text( + """ + #!/bin/bash + get_setting "use_hlebios" + HLEBIOS='0' + BIOS="" + if [ "${HLEBIOS}" != 1 ]; then + if [ -e "/storage/roms/bios/saturn_bios.bin" ]; then + BIOS="-b /storage/roms/bios/saturn_bios.bin" + else + text_viewer -e -w -t "Notice! Yabasanshiro BIOS Missing!" -m "not found" + fi + fi + { yabasanshiro -r 2 -i "${1}" ${BIOS}; } > /tmp/logs/exec.log 2>&1 + """ +) + +PICO8 = text( + """ + #!/bin/bash + mkdir -p "/storage/roms/pico-8" + if [ ! -f "/storage/roms/pico-8/pico8_64" ] || [ ! -f "/storage/roms/pico-8/pico8.dat" ]; then + text_viewer -e -w -t "Missing Pico-8 binaries!" -m "Extract your purchased pico8_64." + else + /storage/roms/pico-8/pico8_64 -home -root_path /storage/roms/pico-8 ${OPTIONS} "${CART}" + fi + """ +) + +PIEMU = text( + """ + #!/bin/bash + ROMS_DIR="/storage/roms" + BIOS_DIR="$ROMS_DIR/bios" + if [[ "$EXTENSION" == "pfs" ]]; then + /usr/bin/mkpfi "$BIOS_DIR/all.bin" + fi + """ +) + +HATARI_SH = text( + """ + #!/bin/bash + HATARI_CONFIG_DIR=/storage/.config/hatari/conf + cp $HATARI_CONFIG_DIR/Atari-ST-default.cfg "$HATARI_TMP_CONFIG" + nice -n -19 hatari --configfile "$HATARI_TMP_CONFIG" --disk-a "$1" >> $HATARI_LOG 2>&1 + """ +) + +HATARI_CFG = text( + """ + [ROM] + szTosImageFileName = /storage/roms/bios/tos.img + bPatchTos = TRUE + """ +) + +ATARI800_CFG = text( + """ + Atari 800 Emulator, Version 3.1.0 + ROM_OS_A_NTSC= + ROM_OS_A_PAL=/storage/roms/bios/ATARIOSA.ROM + ROM_5200=/storage/roms/bios/5200.rom + """ +) + +ATARI800_PATCH = text( + """ + + LDFLAGS += -static-libgcc + + CFLAGS += -DDEFAULT_CFG_NAME="\\"/storage/.config/distribution/configs/atari800.cfg\\"" + """ +) + +AMIBERRY_MK = text( + """ + makeinstall_target() { + mkdir -p ${INSTALL}/usr/config/amiberry + ln -s /storage/roms/bios ${INSTALL}/usr/config/amiberry/kickstarts + } + """ +) + +AMIBERRY_SH = text( + """ + #!/bin/bash + AMIBERRY_CONFIG_DIR=$AMIBERRY_DIR/conf + AMIBERRY_SET_CONF() { + if [ `echo $1 | egrep 'AGA|CD32' | wc -l` -eq 1 ]; then + cp $AMIBERRY_CONFIG_DIR/AmigaA1200-default.uae "$AMIBERRY_TMP_CONFIG" + else + cp $AMIBERRY_CONFIG_DIR/AmigaA500-default.uae "$AMIBERRY_TMP_CONFIG" + fi + } + """ +) + + +def uae(rom: str) -> str: + return f"amiberry.rom_path=/storage/.config/amiberry/kickstarts/\nkickstart_rom_file=$(FILE_PATH)/{rom}\n" + + +SCUMMVM_SH = text( + """ + #!/bin/bash + case $1 in + "libretro") + sed -i '2,2s/^/themepath=\\/roms\\/bios\\/scummvm\\/theme\\n/' ${BIOSPATH}/scummvm.ini + sed -i '2,2s/^/extrapath=\\/roms\\/bios\\/scummvm\\/extra\\n/' ${BIOSPATH}/scummvm.ini + sed -i '2,2s/^/browser_lastpath=\\/roms\\/bios\\/scummvm\\/extra\\n/' ${BIOSPATH}/scummvm.ini + ;; + esac + """ +) + +FREEJ2ME = text( + """ + #!/bin/bash + if [ ! -f "/storage/roms/bios/{jar}" ]; then + cp /usr/config/distribution/freej2me/{jar} /storage/roms/bios + fi + """ +) + +EASYRPG = text( + """ + #!/bin/bash + install_bios() { + BIOS="$1" + curl -Lo "${BIOS}" "${PKG_URL}/${BIOS}" + } + INSTALLPATH="/storage/roms/bios" + PKG_URL="https://dl.degica.com/rpgmakerweb/run-time-packages" + install_bios rpg2000_rtp_installer.exe rtp/2000 + """ +) + + +def tree(**overrides: str) -> dict[str, str]: + files = { + al.RUNEMU: RUNEMU, + al.RETROARCH_CFG: 'savefile_directory = "~/saves"\nsystem_directory = "~/roms/bios"\n', + al.RETRORUN: '/usr/bin/retrorun -c "${RRCONF}" -s /storage/roms/"$3" -d /roms/bios "$EMU" "$2"\n', + al.SCRIPTS["yabasanshiroSA"]: YABASANSHIRO, + al.SCRIPTS["pico8"]: PICO8, + al.SCRIPTS["piemu"]: PIEMU, + al.SCRIPTS["HATARISA"]: HATARI_SH, + al.HATARI_CONFIG: HATARI_CFG, + al.ATARI800_CONFIG: ATARI800_CFG, + al.ATARI800_PATCH: ATARI800_PATCH, + al.AMIBERRY_PACKAGE: AMIBERRY_MK, + al.SCRIPTS["AMIBERRY"]: AMIBERRY_SH, + f"{al.AMIBERRY_CONFIGS}/AmigaA1200-default.uae": uae("kick40068.A1200.rom"), + f"{al.AMIBERRY_CONFIGS}/AmigaA500-default.uae": uae("kick34005.A500.rom"), + al.SCRIPTS["SCUMMVMSA"]: SCUMMVM_SH, + al.EASYRPG: EASYRPG, + **{path: FREEJ2ME.replace("{jar}", f"{core}-lr.jar") for core, path in al.FREEJ2ME}, + } + files.update(overrides) + return files + + +def read(**overrides: str): + return tree(**overrides).__getitem__ + + +class Frontends(unittest.TestCase): + def test_retroarch_system_directory(self): + (record,) = al.retroarch(read()) + self.assertEqual(record.search_dir, "~/roms/bios") + self.assertEqual(record.source_ref, f"{al.RETROARCH_CFG}:2") + + def test_retrorun_directory(self): + (record,) = al.retrorun(read()) + self.assertEqual((record.search_dir, record.passes_to), ("/roms/bios", "command line -d")) + + def test_mame_search_path_of_both_commands(self): + (record,) = al.mame(al.Runemu(read())) + self.assertEqual(record.search_dir, "/roms/bios/mame;/roms/bios") + self.assertEqual(record.depends_on_option, "self.emulator == 'mame'") + self.assertEqual(record.source_ref, f"{al.RUNEMU}:11, 13, 15") + + def test_a_retroarch_config_without_the_key_raises(self): + with self.assertRaises(LauncherShapeError): + al.retroarch(read(**{al.RETROARCH_CFG: "x = 1\n"})) + + +class Standalone(unittest.TestCase): + def test_yabasanshiro_flag_after_the_presence_test(self): + (record,) = al.yabasanshiro(read(), al.Runemu(read())) + self.assertEqual(record.names, ["saturn_bios.bin"]) + self.assertEqual(record.passes_to, "command line -b") + self.assertIs(record.fails_when_missing, False) + self.assertEqual(record.depends_on_option, "use_hlebios") + self.assertEqual( + record.source_ref, + f"{al.RUNEMU}:6, {al.SCRIPTS['yabasanshiroSA']}:2, 5-7, 12", + ) + + def test_pico8_binaries_outside_bios(self): + (record,) = al.pico8(read(), al.Runemu(read())) + self.assertEqual(record.names, ["pico8_64", "pico8.dat"]) + self.assertEqual(record.search_dir, "/storage/roms/pico-8") + self.assertEqual(record.fails_when_missing, "Missing Pico-8 binaries!") + self.assertEqual(record.source_ref, f"{al.RUNEMU}:4, {al.SCRIPTS['pico8']}:3-6") + + def test_piemu_flash_image_source(self): + (record,) = al.piemu(read(), al.Runemu(read())) + self.assertEqual(record.names, ["all.bin"]) + self.assertEqual(record.source_ref, f"{al.RUNEMU}:5, {al.SCRIPTS['piemu']}:2-5") + + def test_hatari_default_config(self): + (record,) = al.hatari(read(), al.Runemu(read())) + self.assertEqual(record.names, ["tos.img"]) + self.assertEqual(record.passes_to, "Atari-ST-default.cfg [ROM] szTosImageFileName") + self.assertEqual( + record.source_ref, + f"{al.RUNEMU}:3, {al.SCRIPTS['HATARISA']}:3-4, {al.HATARI_CONFIG}:2", + ) + + def test_amiberry_link_and_one_kickstart_per_config(self): + link, a1200, a500 = al.amiberry(read(), al.Runemu(read())) + self.assertEqual(link.kind, "links") + self.assertEqual(link.passes_to, "/usr/config/amiberry/kickstarts is a symlink to it") + self.assertEqual((a1200.slot, a1200.names), ("A1200", ["kick40068.A1200.rom"])) + self.assertEqual((a500.slot, a500.names), ("A500", ["kick34005.A500.rom"])) + self.assertEqual(a500.depends_on_option, f"not ({a1200.depends_on_option})") + + def test_a_table_running_another_script_raises(self): + broken = RUNEMU.replace("/usr/bin/yabasanshiro.sh", "/usr/bin/yaba.sh") + with self.assertRaises(LauncherShapeError): + al.yabasanshiro(read(), al.Runemu(read(**{al.RUNEMU: broken}))) + + def test_a_bios_test_written_another_way_raises(self): + broken = YABASANSHIRO.replace('if [ -e "/storage', 'if test -e "/storage') + with self.assertRaises(LauncherShapeError): + al.yabasanshiro(read(**{al.SCRIPTS["yabasanshiroSA"]: broken}), al.Runemu(read())) + + +class Cores(unittest.TestCase): + def test_atari800_built_to_read_the_shipped_config(self): + (record,) = al.atari800(read()) + self.assertEqual(record.names, ["ATARIOSA.ROM", "5200.rom"]) + self.assertEqual(record.source_ref, f"{al.ATARI800_PATCH}:2, {al.ATARI800_CONFIG}:3-4") + + def test_atari800_built_for_another_config_raises(self): + broken = ATARI800_PATCH.replace("atari800.cfg", "other.cfg") + with self.assertRaises(LauncherShapeError): + al.atari800(read(**{al.ATARI800_PATCH: broken})) + + def test_scummvm_paths_written_under_bios(self): + theme, extra = al.scummvm(read()) + self.assertEqual(theme.search_dir, "/roms/bios/scummvm/theme") + self.assertEqual(extra.passes_to, "scummvm.ini extrapath") + + def test_freej2me_copies_the_shipped_jar(self): + plain, plus = al.freej2me(read()) + self.assertEqual((plain.kind, plain.names), ("copies", ["freej2me-lr.jar"])) + self.assertEqual(plus.names, ["freej2me-plus-lr.jar"]) + self.assertEqual(plain.search_dir, "/usr/config/distribution/freej2me") + + def test_easyrpg_download_per_runtime(self): + (record,) = al.easyrpg(read()) + self.assertEqual( + record.search_dir, + "https://dl.degica.com/rpgmakerweb/run-time-packages/rpg2000_rtp_installer.exe", + ) + self.assertEqual(record.passes_to, "/storage/roms/bios/rtp/2000") + + def test_every_extractor_runs_on_the_fixture(self): + records = al.extract_launchers(read()) + # One easyrpg runtime in the fixture, two at the pinned commit. + self.assertEqual(len(records), 16) + + +class PlatformFile(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.config = yaml.safe_load((REPO_ROOT / "platforms" / "amberelec.yml").read_text()) + + def test_the_layer_is_written(self): + self.assertTrue(self.config["launchers"]) + + def test_every_reference_is_in_the_repository(self): + for record in self.config["launchers"]: + self.assertTrue(record["source_ref"].startswith("packages/"), record) + + def test_every_emulator_is_one_amberelec_names(self): + cores = {*self.config["cores"], *self.config.get("standalone_cores", [])} + named = {n for record in self.config["launchers"] for n in record["emulator"].split(", ")} + # RetroArch and retrorun themselves, the two runners of every core. + self.assertEqual(sorted(named - cores), ["retroarch", "retrorun"]) + + +if __name__ == "__main__": + unittest.main() From cedeee225444f9f7335e53c473f94f59a2d6989e Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:58:37 +0100 Subject: [PATCH 4/7] refactor: one meaning for copy records A copies record names its files under search_dir, the folder the code copies from, and passes_to is the target, as the schema and RetroPie, RetroDECK and AmberELEC already write it. KNULLI listed patches.zip under the BIOS folder it is copied into, and KNULLI and RetroBat wrote the target as prose with the condition. The condition of a copy now goes in depends_on_option. Only launcher records change in knulli.yml and retrobat.yml. --- platforms/knulli.yml | 17 +++++---- platforms/retrobat.yml | 50 +++++++++++++++------------ schemas/platform.schema.json | 2 +- scripts/scraper/knulli_launchers.py | 37 ++++++++++++++------ scripts/scraper/retrobat_launchers.py | 45 +++++++++++++++--------- tests/test_launchers_knulli.py | 5 +-- tests/test_launchers_retrobat.py | 10 ++++-- wiki/adding-a-platform.md | 3 +- 8 files changed, 105 insertions(+), 64 deletions(-) diff --git a/platforms/knulli.yml b/platforms/knulli.yml index 1ca669064..5d443be22 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -5793,29 +5793,32 @@ launchers: - switch/prod.keys case_insensitive: false search_dir: /userdata/bios - passes_to: copied to /userdata/system/configs/Ryujinx/system/prod.keys + passes_to: /userdata/system/configs/Ryujinx/system/prod.keys fails_when_missing: false + depends_on_option: "ryujinxKeys.exists()" source_ref: "package/system/knulli-configgen/configgen/configgen/generators/ryujinx/ryujinxGenerator.py:107-109" - emulator: pcsx2 build: standalone kind: copies names: - - ps2/patches.zip + - patches.zip case_insensitive: false - search_dir: /userdata/bios - passes_to: copied from /usr/share/knulli/datainit/bios/ps2/patches.zip + search_dir: /usr/share/knulli/datainit/bios/ps2 + passes_to: /userdata/bios/ps2/patches.zip fails_when_missing: false + depends_on_option: "not pcsx2Patches.exists()" source_ref: "package/system/knulli-configgen/configgen/configgen/generators/pcsx2/pcsx2Generator.py:78,\ \ 121-122" - emulator: aethersx2 build: standalone kind: copies names: - - ps2/patches.zip + - patches.zip case_insensitive: false - search_dir: /userdata/bios - passes_to: copied from /usr/share/knulli/datainit/bios/ps2/patches.zip + search_dir: /usr/share/knulli/datainit/bios/ps2 + passes_to: /userdata/bios/ps2/patches.zip fails_when_missing: false + depends_on_option: "not pcsx2Patches.exists()" source_ref: "package/system/knulli-configgen/configgen/configgen/generators/aethersx2/aethersx2Generator.py:59,\ \ 93-94" - emulator: gsplus diff --git a/platforms/retrobat.yml b/platforms/retrobat.yml index 7cbb89ea0..736923af7 100644 --- a/platforms/retrobat.yml +++ b/platforms/retrobat.yml @@ -3880,9 +3880,10 @@ launchers: - capsimg.dll case_insensitive: true search_dir: bios - passes_to: copied to /, only when the target does not exist + passes_to: / fails_when_missing: false - depends_on_option: core in capsimgCore, rom extension ".ipf" + depends_on_option: core in capsimgCore, rom extension ".ipf", the target does not + exist source_ref: "emulatorLauncher/Generators/LibRetro.Generator.cs:1018-1027, 2750" - emulator: demul build: standalone @@ -3943,11 +3944,11 @@ launchers: - segaboot.gcm case_insensitive: true search_dir: bios - passes_to: first found copied to /User/Triforce/segaboot.gcm, skipped - when the target is as large and not older + passes_to: /User/Triforce/segaboot.gcm fails_when_missing: "warning '[WARNING] Triforce: segaboot.gcm not found in bios\\\ triforce, test menu will not be available.' when the target is absent too" - depends_on_option: _triforce, !_crediar + depends_on_option: _triforce, !_crediar, first name found, the target is absent, + smaller or older source_ref: "emulatorLauncher/Generators/Dolphin.Generator.cs:782-783, 1048-1083" - emulator: dosbox-pure build: standalone @@ -3959,8 +3960,9 @@ launchers: - Roland_SC-88.sf2 case_insensitive: true search_dir: bios/scummvm/extra - passes_to: copied to /system/, only when the target does not exist + passes_to: /system/ fails_when_missing: false + depends_on_option: the target does not exist source_ref: "emulatorLauncher/Generators/DosBoxPure.Generator.cs:125, 130-148" - emulator: dosbox-staging build: standalone @@ -3970,9 +3972,9 @@ launchers: - MT32_PCM.ROM case_insensitive: true search_dir: bios/scummvm/extra - passes_to: copied to /mt32-roms/, only when the target does not - exist + passes_to: /mt32-roms/ fails_when_missing: false + depends_on_option: the target does not exist source_ref: "emulatorLauncher/Generators/DosBoxStaging.Generator.cs:321, 326-334" - emulator: dosbox-staging build: standalone @@ -3982,9 +3984,9 @@ launchers: - Roland_SC-88.sf2 case_insensitive: true search_dir: bios/scummvm/extra - passes_to: copied to /soundfonts/, only when the target does not - exist + passes_to: /soundfonts/ fails_when_missing: false + depends_on_option: the target does not exist source_ref: "emulatorLauncher/Generators/DosBoxStaging.Generator.cs:318, 336-344" - emulator: duckstation build: standalone @@ -4426,9 +4428,9 @@ launchers: names: *id001 case_insensitive: true search_dir: bios/hatari/tos - passes_to: copied to bios/hatarib/, only when the target does not exist + passes_to: bios/hatarib/ fails_when_missing: false - depends_on_option: hatarib_tos + depends_on_option: hatarib_tos, the target does not exist source_ref: "emulatorLauncher/Generators/LibRetro.CoreOptions.cs:3434-3435, 3457-3460" - emulator: jzintv build: standalone @@ -4620,9 +4622,9 @@ launchers: - syscard3.pce case_insensitive: true search_dir: bios - passes_to: "copied to /Firmware/[BIOS] Super CD-ROM System (Japan) (v3.0).pce,\ - \ only when the target does not exist" + passes_to: "/Firmware/[BIOS] Super CD-ROM System (Japan) (v3.0).pce" fails_when_missing: "ApplicationException 'PCE CD firmware is missing (syscard3.pce)'" + depends_on_option: the target does not exist source_ref: "emulatorLauncher/Generators/Mesen.Generator.cs:254-269" - emulator: mesen build: standalone @@ -4632,10 +4634,10 @@ launchers: - SGB2.sfc case_insensitive: true search_dir: bios - passes_to: copied to /Firmware/SGB1.sfc, SGB2.sfc, only when the target does - not exist + passes_to: /Firmware/SGB1.sfc, /Firmware/SGB2.sfc fails_when_missing: "ApplicationException 'Super Gameboy firmware is missing (SGB1.sfc\ \ and/or SGB2.sfc)'" + depends_on_option: the target does not exist source_ref: "emulatorLauncher/Generators/Mesen.Generator.cs:281-304" - emulator: mesen build: standalone @@ -4644,9 +4646,9 @@ launchers: - gba_bios.bin case_insensitive: true search_dir: bios - passes_to: copied to /Firmware/gba_bios.bin, only when the target does not - exist + passes_to: /Firmware/gba_bios.bin fails_when_missing: false + depends_on_option: the target does not exist source_ref: "emulatorLauncher/Generators/Mesen.Generator.cs:307-324" - emulator: mesen build: standalone @@ -4655,8 +4657,9 @@ launchers: - colecovision.rom case_insensitive: true search_dir: bios - passes_to: copied to /Firmware/bios.col, only when the target does not exist + passes_to: /Firmware/bios.col fails_when_missing: "ApplicationException 'Colecovision firmware is missing (colecovision.rom)'" + depends_on_option: the target does not exist source_ref: "emulatorLauncher/Generators/Mesen.Generator.cs:326-346" - emulator: openmsx build: standalone @@ -5269,9 +5272,9 @@ launchers: - kick40068.A4000 case_insensitive: true search_dir: bios, bios/amiga, bios/Amiga - passes_to: copied to /Devs/Kickstarts/ from the system's Kickstart - candidates + passes_to: /Devs/Kickstarts/ fails_when_missing: false + depends_on_option: the system's Kickstart candidates source_ref: "emulatorLauncher/Generators/Uae.Generator.cs:1002-1011, 1026-1032" - emulator: xemu build: standalone @@ -5343,9 +5346,10 @@ launchers: - IPLROM30.DAT case_insensitive: true search_dir: bios/keropi - passes_to: copied to /, only when the target does not exist + passes_to: / fails_when_missing: false - depends_on_option: "!string.IsNullOrEmpty(biosRoot), Directory.Exists(biosPath)" + depends_on_option: "!string.IsNullOrEmpty(biosRoot), Directory.Exists(biosPath),\ + \ the target does not exist" source_ref: "emulatorLauncher/Generators/Xm6pro.Generator.cs:118-141" - emulator: xroar build: standalone diff --git a/schemas/platform.schema.json b/schemas/platform.schema.json index 28786b8ca..0d409de16 100644 --- a/schemas/platform.schema.json +++ b/schemas/platform.schema.json @@ -226,7 +226,7 @@ "scans_dir", "package" ], - "description": "names_searched: tries the names in order, first found wins. fixed_name: writes or opens one path per name. core_option: sets a core option whose value names the file. passes_path: hands over a directory. links, copies: links or copies a file to where the emulator reads it. moves: moves a file the user left in one folder to the folder the launcher links from. scans_dir: the emulator picks from a directory. package: names no file, records what a package builds (`repo`) and the systems it registers (`systems`)" + "description": "names_searched: tries the names in order, first found wins. fixed_name: writes or opens one path per name. core_option: sets a core option whose value names the file. passes_path: hands over a directory. links, copies: links or copies a file to where the emulator reads it (a copy names its files under search_dir, the folder it copies from, passes_to is the target, and the case that skips it is in depends_on_option). moves: moves a file the user left in one folder to the folder the launcher links from. scans_dir: the emulator picks from a directory. package: names no file, records what a package builds (`repo`) and the systems it registers (`systems`)" }, "package": { "type": "string", diff --git a/scripts/scraper/knulli_launchers.py b/scripts/scraper/knulli_launchers.py index 8ebc0fd59..219174230 100644 --- a/scripts/scraper/knulli_launchers.py +++ b/scripts/scraper/knulli_launchers.py @@ -332,7 +332,8 @@ class GenericSites: """A generator module whose BIOS path uses bios_sites() reads. kind is fixed_name when the path names a file, passes_path when it names - a directory. A copy is recorded as `copies` whatever the kind. caller + a directory. A copy is recorded as `copies` whatever the kind, its names + read from the folder the code copies from and passes_to the target. caller names the module and function whose core test selects the cores this helper module runs for, emulator the name when no core test does. """ @@ -384,6 +385,8 @@ class _Site: conditions: list[str] cores: list[str] expands: dict[str, list[str]] = field(default_factory=dict) + # A copy reads its file from `source` and hands it to `passes_to`. + source: str | None = None class _Reader: @@ -394,6 +397,7 @@ class _Reader: self.src = src self.bios = cg.bios.rstrip("/") self.parents: dict[int, ast.AST] = {} + self.copy_sources: dict[int, str] = {} for parent in ast.walk(src.tree): for child in ast.iter_child_nodes(parent): self.parents[id(child)] = parent @@ -523,10 +527,14 @@ class _Reader: continue yield node, value # type: ignore[misc] - def conditions(self, node: ast.AST) -> tuple[list[str], list[tuple[int, int]]]: + def conditions( + self, node: ast.AST, guards: bool = False + ) -> tuple[list[str], list[tuple[int, int]]]: """Tests of the ifs a node sits in, outermost first, and their lines. - A presence test of a BIOS path is a guard, kept for its lines only. + A presence test of a BIOS path is a guard, kept for its lines only, + unless guards is set: a copy made only when its source or its target + is there carries the test as a condition. """ tests: list[str] = [] spans: list[tuple[int, int]] = [] @@ -535,7 +543,7 @@ class _Reader: if isinstance(parent, ast.If) and child is not parent.test: spans.append(self.src.span(parent.test)) text = ast.unparse(parent.test) - if not self._is_guard(parent.test): + if guards or not self._is_guard(parent.test): in_else = any(child is stmt for stmt in parent.orelse) tests.append(f"not ({text})" if in_else else text) child, parent = parent, self.parents.get(id(parent)) @@ -660,9 +668,10 @@ class _Reader: receiver = func.value if isinstance(func, ast.Attribute) else None if attr in ("copy", "copyfile", "copy2") and receiver is not None and ast.unparse(receiver) == "shutil": source, target = parent.args[0], parent.args[1] - if source is node: - return f"copied to {self.resolve(target) or ast.unparse(target)}", "copies" - return f"copied from {self.resolve(source) or ast.unparse(source)}", "copies" + self.copy_sources[id(node)] = value if source is node else ( + self.resolve(source) or ast.unparse(source) + ) + return (value if target is node else self.resolve(target) or ast.unparse(target)), "copies" if ( attr == "append" and isinstance(receiver, ast.Name) @@ -718,7 +727,7 @@ class _Reader: def _site( self, read: ast.AST, value: str, passes_to: str, kind: str | None, defined: ast.AST ) -> _Site: - tests, spans = self.conditions(read) + tests, spans = self.conditions(read, guards=kind == "copies") lines = [self.src.span(self.statement_of(read)), *spans] if defined is not read: # A path bound in one branch and read after the if: the branch @@ -726,7 +735,10 @@ class _Reader: bound_tests, bound_spans = self.conditions(defined) tests = list(dict.fromkeys([*bound_tests, *tests])) lines += [self.src.span(self.statement_of(defined)), *bound_spans] - return _Site(value, passes_to, kind, lines, tests, self.cores(read), self._expands(read)) + return _Site( + value, passes_to, kind, lines, tests, self.cores(read), self._expands(read), + self.copy_sources.get(id(read)), + ) def _guard_site(self, read: ast.AST, value: str) -> _Site: """A presence test whose if body is the only consumer of the path.""" @@ -802,7 +814,12 @@ def bios_sites(cg: Configgen, spec: GenericSites) -> list[Launcher]: expands = {k: v for site in group for k, v in site.expands.items()} names: list[str] = [] search_dir = path - if kind in ("fixed_name", "copies"): + source = group[0].source + if kind == "copies" and source is not None and not source.startswith(prefix): + # Copied into BIOS: the file is read from the folder it comes from. + search_dir, _, name = source.rpartition("/") + names = [name] + elif kind in ("fixed_name", "copies"): if not path.startswith(prefix): src.fail(f"{path} names no file under {cg.bios}") names, search_dir = [path[len(prefix):]], cg.bios diff --git a/scripts/scraper/retrobat_launchers.py b/scripts/scraper/retrobat_launchers.py index 6ef1898a5..a3957f340 100644 --- a/scripts/scraper/retrobat_launchers.py +++ b/scripts/scraper/retrobat_launchers.py @@ -320,6 +320,16 @@ def feature_choices( return values, src.lines(feature.start(), close + len("")) +# A copy record reads `names` from `search_dir` and hands them to the path in +# `passes_to`. When the copy is skipped is a condition of the record. +TARGET_ABSENT = "the target does not exist" + + +def _joined(*parts: str | None) -> str | None: + """Conditions of a record, joined, None when there are none.""" + return ", ".join(part for part in parts if part) or None + + def _record( emulator: str, build: str, kind: str, names: list[str], ref: str, **fields: object ) -> Record: @@ -611,9 +621,9 @@ def hatari_libretro(sources: dict[str, Source]) -> list[Record]: source_ref((src.path, [head, src.lines(shared.start(), copy.end())])), case_insensitive=True, search_dir="/".join(["bios", *folders]), - passes_to=f"copied to bios/{own.group(2)}/, only when the target does not exist", + passes_to=f"bios/{own.group(2)}/", fails_when_missing=False, - depends_on_option=key, + depends_on_option=_joined(key, TARGET_ABSENT), ) ) return records @@ -661,10 +671,9 @@ def mesen(sources: dict[str, Source]) -> list[Record]: source_ref((src.path, [src.statement_span(block)])), case_insensitive=True, search_dir="bios", - passes_to="copied to /Firmware/" - + ", ".join(destinations) - + ", only when the target does not exist", + passes_to=", ".join(f"/Firmware/{d}" for d in destinations), fails_when_missing=fails, + depends_on_option=TARGET_ABSENT, ) ) return records @@ -804,11 +813,9 @@ def winuae(sources: dict[str, Source]) -> list[Record]: ), case_insensitive=True, search_dir=search, - passes_to=( - f"copied to /{'/'.join(_strings(folder.group(1)))}/" - " from the system's Kickstart candidates" - ), + passes_to=f"/{'/'.join(_strings(folder.group(1)))}/", fails_when_missing=False, + depends_on_option="the system's Kickstart candidates", ) ) return records @@ -2480,10 +2487,12 @@ def dolphin(sources: dict[str, Source]) -> list[Record]: ), case_insensitive=True, search_dir="bios", - passes_to=f"first found copied to {destination[0] if destination else target.group(1)}," - " skipped when the target is as large and not older", + passes_to=destination[0] if destination else target.group(1), fails_when_missing=f"warning '{_unescape(warning.group(1))}' when the target is absent too", - depends_on_option=", ".join(conditions) or None, + depends_on_option=_joined( + *conditions, "first name found", + "the target is absent, smaller or older", + ), ) ) return _covered(src, records, skip=(r"DolphinTool\.exe",)) @@ -2578,9 +2587,9 @@ def _copies_if_absent( _path_parts(src, *returns, *[s for c in copies for s in c[3]]), case_insensitive=True, search_dir=folders.pop(), - passes_to=f"copied to {destination}/, only when the target does not exist", + passes_to=f"{destination}/", fails_when_missing=False, - depends_on_option=", ".join(conditions) or None, + depends_on_option=_joined(*conditions, TARGET_ABSENT), ) ) if not records: @@ -2619,7 +2628,9 @@ def capsimg(sources: dict[str, Source]) -> list[Record]: record["source_ref"] = source_ref( (src.path, [(src.line(cores.start()), core_lines[1]), src.statement_span(block)]) ) - record["depends_on_option"] = f'core in {cores.group(1)}, rom extension "{gate.group(1)}"' + record["depends_on_option"] = _joined( + f'core in {cores.group(1)}, rom extension "{gate.group(1)}"', TARGET_ABSENT + ) return [record] @@ -3108,9 +3119,9 @@ def xm6pro(sources: dict[str, Source]) -> list[Record]: _path_parts(src, (src.line(table.start()), lines[1]), src.lines(method.start(), body[1])), case_insensitive=True, search_dir=folder[0], - passes_to="copied to /, only when the target does not exist", + passes_to="/", fails_when_missing=False, - depends_on_option=", ".join(conditions) or None, + depends_on_option=_joined(*conditions, TARGET_ABSENT), ) return _covered(src, [record]) diff --git a/tests/test_launchers_knulli.py b/tests/test_launchers_knulli.py index 8ea3070b0..ca0b6e7da 100644 --- a/tests/test_launchers_knulli.py +++ b/tests/test_launchers_knulli.py @@ -451,8 +451,9 @@ class BiosSites(unittest.TestCase): (record,) = sites("ryujinx/ryujinxGenerator.py", "fixed_name") self.assertEqual(record["kind"], "copies") self.assertEqual(record["names"], ["switch/prod.keys"]) - self.assertEqual(record["passes_to"], "copied to /userdata/system/configs/Ryujinx/system/prod.keys") - self.assertNotIn("depends_on_option", record) + self.assertEqual(record["search_dir"], "/userdata/bios") + self.assertEqual(record["passes_to"], "/userdata/system/configs/Ryujinx/system/prod.keys") + self.assertEqual(record["depends_on_option"], "ryujinxKeys.exists()") def test_a_presence_test_over_a_literal_list(self): (record,) = sites("dolphin/dolphinGenerator.py", "fixed_name") diff --git a/tests/test_launchers_retrobat.py b/tests/test_launchers_retrobat.py index 3f33dd13b..23029453f 100644 --- a/tests/test_launchers_retrobat.py +++ b/tests/test_launchers_retrobat.py @@ -1042,7 +1042,10 @@ class HelpersAndCopies(unittest.TestCase): ) self.assertEqual(segaboot["kind"], "copies") self.assertEqual(segaboot["names"], ["triforce/segaboot.gcm", "segaboot.gcm"]) - self.assertEqual(segaboot["depends_on_option"], "_triforce, !_crediar") + self.assertEqual( + segaboot["depends_on_option"], + "_triforce, !_crediar, first name found, the target is absent, smaller or older", + ) def test_dosbox_copies_grouped_by_target_folder(self): sources = _sources( @@ -1055,16 +1058,17 @@ class HelpersAndCopies(unittest.TestCase): ( "dosbox-pure", ["MT32_CONTROL.ROM"], - "copied to /mt32-roms/, only when the target does not exist", + "/mt32-roms/", ), ( "dosbox-pure", ["Roland_SC-55.sf2"], - "copied to /soundfonts/, only when the target does not exist", + "/soundfonts/", ), ], ) self.assertEqual(records[0]["search_dir"], "bios/scummvm/extra") + self.assertEqual(records[0]["depends_on_option"], "the target does not exist") def test_tsugaru_folder_chain(self): [record] = rl.tsugaru(_sources(**{f"{G}/Tsugaru.Generator.cs": TSUGARU})) diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index 3e39160d4..4a426b86c 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -491,7 +491,8 @@ the launcher's name, a file the core accepts: the file the collection knows under that name when the core takes its content, otherwise the copy held for the core entry read under that name. Records that only hand over a directory (`passes_path`, `scans_dir`, a `links` record with no names) or copy a file -the platform already names (`copies`) place nothing, and a `search_dir` the +the platform already names (`copies`, its `names` under the `search_dir` it +copies from, the target in `passes_to`) place nothing, and a `search_dir` the code describes rather than names ("directory of the Chihiro BIOS found") is skipped. `$biosdir`, `${biosPath}` and `$bios_path` are the BIOS directory as RetroPie, EmuDeck and RetroDECK spell it, and a path from a root From f0289e3624eb81fc2e44c7422296c7d4752d8ab5 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:59:09 +0100 Subject: [PATCH 5/7] feat: record pcsx2's relative bios folder configureINI writes [Folders] Bios = ../../../bios/ps2 into inis/PCSX2.ini, which PCSX2 and AetherSX2 resolve against the config folder configgen hands it. relative_folders reads the call, the ini path and the value, and records the resolved /userdata/bios/ps2 as a passes_path for pcsx2 and aethersx2. --- platforms/knulli.yml | 18 +++++++ scripts/scraper/knulli_launchers.py | 77 ++++++++++++++++++++++++++++- tests/test_launchers_knulli.py | 52 +++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) diff --git a/platforms/knulli.yml b/platforms/knulli.yml index 5d443be22..4e25c2144 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -5732,6 +5732,24 @@ launchers: depends_on_option: neocd_bios source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroOptions.py:2558-2559,\ \ 2566-2569" +- emulator: pcsx2 + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/ps2 + passes_to: "inis/PCSX2.ini [Folders] Bios = ../../../bios/ps2, relative to /userdata/system/configs/PCSX2" + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/pcsx2/pcsx2Generator.py:92,\ + \ 176, 213" +- emulator: aethersx2 + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/ps2 + passes_to: "inis/PCSX2.ini [Folders] Bios = ../../../bios/ps2, relative to /userdata/system/.config/aethersx2" + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/aethersx2/aethersx2Generator.py:71,\ + \ 182, 219" - emulator: lexaloffle build: standalone kind: moves diff --git a/scripts/scraper/knulli_launchers.py b/scripts/scraper/knulli_launchers.py index 219174230..a01cd8a8c 100644 --- a/scripts/scraper/knulli_launchers.py +++ b/scripts/scraper/knulli_launchers.py @@ -20,6 +20,7 @@ LauncherShapeError, as every other extractor does. from __future__ import annotations import ast +import posixpath import re from collections.abc import Callable, Iterator from dataclasses import dataclass, field @@ -834,8 +835,82 @@ def bios_sites(cg: Configgen, spec: GenericSites) -> list[Launcher]: return launchers +# Generators that write a [Folders] path relative to the config folder they +# are given. PCSX2 and AetherSX2 resolve those paths against their data root, +# the folder holding inis/, which configureINI receives as config_directory. +RELATIVE_FOLDERS = ( + ("pcsx2/pcsx2Generator.py", "configureINI"), + ("aethersx2/aethersx2Generator.py", "configureINI"), +) + + +def relative_folders( + cg: Configgen, specs: tuple[tuple[str, str], ...] = RELATIVE_FOLDERS +) -> list[Launcher]: + """parser.set(section, key, "../") resolved against config_directory.""" + launchers = [] + for path, function_name in specs: + src = cg.source(f"{cg.generators}/{path}") + reader = _Reader(cg, src) + function = src.function(function_name) + if not function.args.args: + src.fail(f"{function_name} takes no config folder") + folder = function.args.args[0].arg + ini = [ + node for node in ast.walk(function) + if isinstance(node, ast.Assign) and isinstance(node.value, ast.BinOp) + and ast.unparse(node.value).startswith(f"{folder} /") + and ast.unparse(node.value).endswith(".ini'") + ] + if len(ini) != 1: + src.fail(f"{function_name} builds no one ini path under {folder}") + parts = sorted( + (n for n in ast.walk(ini[0].value) + if isinstance(n, ast.Constant) and isinstance(n.value, str)), + key=lambda n: (n.lineno, n.col_offset), + ) + ini_name = posixpath.join(*(str(n.value) for n in parts)) + calls = [ + node for node in ast.walk(src.tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + and node.func.id == function_name and node.args + ] + if len(calls) != 1: + src.fail(f"expected one call of {function_name}, found {len(calls)}") + base = reader.resolve(calls[0].args[0]) + if base is None or not base.startswith("/"): + src.fail(f"{function_name} is given a folder that is not a path") + found = 0 + for call in ast.walk(function): + if not ( + isinstance(call, ast.Call) and isinstance(call.func, ast.Attribute) + and call.func.attr == "set" and len(call.args) == 3 + and all(isinstance(a, ast.Constant) for a in call.args) + ): + continue + section, key, value = (str(a.value) for a in call.args) # type: ignore[attr-defined] + if not value.startswith("../"): + continue + resolved = posixpath.normpath(posixpath.join(base, value)) + if not reader.under_bios(resolved): + continue + found += 1 + launchers.append(Launcher( + emulator=cg.emulator(path.removesuffix(".py").replace("/", ".")), + build="standalone", kind="passes_path", names=[], search_dir=resolved, + passes_to=f"{ini_name} [{section}] {key} = {value}, relative to {base}", + fails_when_missing=False, + source_ref=src.ref([ + src.span(reader.statement_of(calls[0])), src.span(ini[0]), _lines(call), + ]), + )) + if not found: + src.fail(f"{function_name} writes no relative folder under {cg.bios}") + return launchers + + EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = ( - *SHARED, lexaloffle, core_file_options, + *SHARED, lexaloffle, core_file_options, relative_folders, ) diff --git a/tests/test_launchers_knulli.py b/tests/test_launchers_knulli.py index ca0b6e7da..9e235fbec 100644 --- a/tests/test_launchers_knulli.py +++ b/tests/test_launchers_knulli.py @@ -59,6 +59,7 @@ IMPORTER = text( 'openmsx': ('openmsx.openmsxGenerator', 'OpenmsxGenerator'), 'hatari': ('hatari.hatariGenerator', 'HatariGenerator'), 'melonds': ('melonds.melondsGenerator', 'MelonDSGenerator'), + 'pcsx2': ('pcsx2.pcsx2Generator', 'Pcsx2Generator'), } """ ) @@ -305,6 +306,29 @@ MELONDS = text( ) +# pcsx2Generator.py: the default patches copied into BIOS when absent, and +# the BIOS folder written relative to the config folder. +PCSX2 = text( + """ + from ...batoceraPaths import BIOS, CONFIGS, DATAINIT_DIR + + _PCSX2_CONFIG: Final = CONFIGS / "PCSX2" + _PCSX2_BIOS: Final = BIOS / "ps2" + + class Pcsx2Generator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + pcsx2Patches = _PCSX2_BIOS / "patches.zip" + configureINI(_PCSX2_CONFIG, _PCSX2_BIOS, system) + if not pcsx2Patches.exists(): + shutil.copy(DATAINIT_DIR / "bios" / "ps2" / "patches.zip", pcsx2Patches) + + def configureINI(config_directory, bios_directory, system): + configFileName = config_directory / 'inis' / "PCSX2.ini" + pcsx2INIConfig.set("Folders", "Bios", "../../../bios/ps2") + pcsx2INIConfig.set("Folders", "Logs", "../../logs") + """ +) + def tree(**overrides: str) -> dict[str, str]: files = { f"{kl.CONFIGGEN}/batoceraPaths.py": PATHS, @@ -322,6 +346,7 @@ def tree(**overrides: str) -> dict[str, str]: f"{G}/libretro/libretroMAMEConfig.py": MAME_CONFIG, f"{G}/hatari/hatariGenerator.py": HATARI, f"{G}/melonds/melondsGenerator.py": MELONDS, + f"{G}/pcsx2/pcsx2Generator.py": PCSX2, } files.update(overrides) return files @@ -498,6 +523,33 @@ class BiosSites(unittest.TestCase): kl.bios_sites(kl.configgen(files.__getitem__), spec) +class CopiesAndFolders(unittest.TestCase): + def test_a_copy_into_bios_reads_from_the_folder_it_comes_from(self): + (record,) = sites("pcsx2/pcsx2Generator.py", "fixed_name") + self.assertEqual(record["kind"], "copies") + self.assertEqual(record["names"], ["patches.zip"]) + self.assertEqual(record["search_dir"], "/usr/share/knulli/datainit/bios/ps2") + self.assertEqual(record["passes_to"], "/userdata/bios/ps2/patches.zip") + self.assertEqual(record["depends_on_option"], "not pcsx2Patches.exists()") + + def test_a_relative_folder_resolved_against_the_config_folder(self): + (record,) = kl.relative_folders(configgen(), kl.RELATIVE_FOLDERS[:1]) + self.assertEqual(record.kind, "passes_path") + self.assertEqual(record.search_dir, "/userdata/bios/ps2") + self.assertEqual( + record.passes_to, + "inis/PCSX2.ini [Folders] Bios = ../../../bios/ps2, relative to /userdata/system/configs/PCSX2", + ) + self.assertEqual(record.source_ref, f"{G}/pcsx2/pcsx2Generator.py:9, 14-15") + + def test_a_folder_written_outside_bios_raises(self): + broken = PCSX2.replace("../../../bios/ps2", "../../../saves/ps2") + with self.assertRaises(LauncherShapeError): + kl.relative_folders( + configgen(**{f"{G}/pcsx2/pcsx2Generator.py": broken}), kl.RELATIVE_FOLDERS[:1] + ) + + class PlatformFile(unittest.TestCase): @classmethod def setUpClass(cls) -> None: From 7ca7872fdc19be491ebeffa1c14bfab01f2b189a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:03:08 +0100 Subject: [PATCH 6/7] feat: record knulli's macintosh boot disk MAME and libretro MAME pass /userdata/bios/.img with -flop1 when the option is in the generator's list, else .chd with -hard. Each branch is a {bootdisk} template: the list the code tests for the floppy, and for the hard disk the choices es_features.yml offers that the list does not take. The menu offers macos3 while the code tests macos30, so System 3.0 is passed as macos3.chd. --- platforms/knulli.yml | 82 ++++++++++++++++ scripts/scraper/knulli_launchers.py | 141 ++++++++++++++++++++++++++++ tests/test_launchers_knulli.py | 63 +++++++++++++ 3 files changed, 286 insertions(+) diff --git a/platforms/knulli.yml b/platforms/knulli.yml index 4e25c2144..3345c487d 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -5764,6 +5764,88 @@ launchers: fails_when_missing: Please add the Raspberry Pi version of Pico-8 to roms/pico8 and try again. source_ref: "board/fsoverlay/usr/bin/knulli-install-pico8:3-11, 16-19, 66-69" +- emulator: mame + build: standalone + kind: fixed_name + names: + - "{bootdisk}.img" + expands: + bootdisk: + - macos30 + - macos608 + - macos701 + - macos75 + case_insensitive: false + search_dir: /userdata/bios + passes_to: command line -flop1 + fails_when_missing: false + depends_on_option: "bootdisk (not (messSysName[messMode] == '' or messMode == -1)\ + \ and softList == '' and system.name == 'macintosh' and system.isOptSet('bootdisk')\ + \ and system.config['bootdisk'] in ['macos30', 'macos608', 'macos701', 'macos75'])" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:303-305,\ + \ 375-379, 381, 385" +- emulator: mame + build: standalone + kind: fixed_name + names: + - "{bootdisk}.chd" + expands: + bootdisk: + - macos3 + - mac608 + - mac701 + - mac755 + case_insensitive: false + search_dir: /userdata/bios + passes_to: command line -hard + fails_when_missing: false + depends_on_option: "bootdisk (not (messSysName[messMode] == '' or messMode == -1)\ + \ and softList == '' and system.name == 'macintosh' and system.isOptSet('bootdisk')\ + \ and not (system.config['bootdisk'] in ['macos30', 'macos608', 'macos701', 'macos75']))" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/mame/mameGenerator.py:303-305,\ + \ 375-379, 384-385, package/emulationstation/knulli-es-system/es_features.yml:11037-11043" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: fixed_name + names: + - "{bootdisk}.img" + expands: + bootdisk: + - macos30 + - macos608 + - macos701 + - macos75 + case_insensitive: false + search_dir: /userdata/bios + passes_to: command line -flop1 + fails_when_missing: false + depends_on_option: "bootdisk (not (system.name in ['mame', 'neogeo', 'lcdgames',\ + \ 'plugnplay', 'vis']) and not (messSysName[messMode] == '') and not (softList\ + \ != '') and system.name == 'macintosh' and system.isOptSet('bootdisk') and system.config['bootdisk']\ + \ in ['macos30', 'macos608', 'macos701', 'macos75'])" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:64,\ + \ 123, 197, 265-268, 270, 274" +- emulator: mame, mess, mamevirtual, same_cdi + build: libretro + kind: fixed_name + names: + - "{bootdisk}.chd" + expands: + bootdisk: + - macos3 + - mac608 + - mac701 + - mac755 + case_insensitive: false + search_dir: /userdata/bios + passes_to: command line -hard + fails_when_missing: false + depends_on_option: "bootdisk (not (system.name in ['mame', 'neogeo', 'lcdgames',\ + \ 'plugnplay', 'vis']) and not (messSysName[messMode] == '') and not (softList\ + \ != '') and system.name == 'macintosh' and system.isOptSet('bootdisk') and not\ + \ (system.config['bootdisk'] in ['macos30', 'macos608', 'macos701', 'macos75']))" + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:64,\ + \ 123, 197, 265-268, 273-274, package/emulationstation/knulli-es-system/es_features.yml:3727-3733" - emulator: dolphin build: standalone kind: fixed_name diff --git a/scripts/scraper/knulli_launchers.py b/scripts/scraper/knulli_launchers.py index a01cd8a8c..5a31bcd6b 100644 --- a/scripts/scraper/knulli_launchers.py +++ b/scripts/scraper/knulli_launchers.py @@ -26,6 +26,8 @@ from collections.abc import Callable, Iterator from dataclasses import dataclass, field from typing import Any +import yaml + try: from .batocera_launchers import ( Configgen, @@ -79,6 +81,7 @@ CONFIGGEN = "package/system/knulli-configgen/configgen/configgen" GENERATORS = f"{CONFIGGEN}/generators" LIBRETRO_OPTIONS = f"{GENERATORS}/libretro/libretroOptions.py" PICO8_INSTALLER = "board/fsoverlay/usr/bin/knulli-install-pico8" +ES_FEATURES = "package/emulationstation/knulli-es-system/es_features.yml" # Batocera's extractors that read KNULLI's generators unchanged. SHARED = ( @@ -909,6 +912,143 @@ def relative_folders( return launchers +def _yaml_node(root: yaml.Node, keys: tuple[str, ...], path: str) -> yaml.Node: + node = root + for key in keys: + if not isinstance(node, yaml.MappingNode): + raise LauncherShapeError(f"{path}: {key} is not under a mapping") + found = [v for k, v in node.value if isinstance(k, yaml.ScalarNode) and k.value == key] + if len(found) != 1: + raise LauncherShapeError(f"{path}: no {'/'.join(keys)}") + node = found[0] + return node + + +def option_choices(read: Callable[[str], str], keys: tuple[str, ...]) -> tuple[list[str], tuple[int, int]]: + """The values es_features.yml offers for an option, and their lines.""" + node = _yaml_node(yaml.compose(read(ES_FEATURES)), (*keys, "choices"), ES_FEATURES) + if not isinstance(node, yaml.MappingNode) or not node.value: + raise LauncherShapeError(f"{ES_FEATURES}: {'/'.join(keys)} has no choices") + values = [str(v.value) for _k, v in node.value if isinstance(v, yaml.ScalarNode)] + return values, (node.start_mark.line + 1, node.end_mark.line) + + +@dataclass(frozen=True) +class BootDisk: + """A generator that names a BIOS file from an option value, and the ES feature offering it.""" + + path: str + feature: tuple[str, ...] + build: str = "standalone" + caller: tuple[str, str] | None = None + + +BOOT_DISKS = ( + BootDisk("mame/mameGenerator.py", ("mame", "systems", "macintosh", "cfeatures")), + BootDisk( + "libretro/libretroMAMEConfig.py", + ("libretro", "cores", "mame", "systems", "macintosh", "cfeatures"), "libretro", + ("libretro/libretroConfig.py", "generateMAMEConfigs"), + ), +) + + +def _option_template(reader: _Reader, node: ast.AST) -> tuple[str, str] | None: + """(option, name) for '/' + system.config[option] + ''.""" + if not (isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add)): + return None + left, suffix = node.left, node.right + if not ( + isinstance(left, ast.BinOp) and isinstance(left.op, ast.Add) + and isinstance(left.left, ast.Constant) and isinstance(suffix, ast.Constant) + and isinstance(left.right, ast.Subscript) + and ast.unparse(left.right.value) == "system.config" + and isinstance(left.right.slice, ast.Constant) + ): + return None + prefix = str(left.left.value).strip('"') + if not prefix.startswith(reader.bios + "/"): + return None + option = str(left.right.slice.value) + name = prefix[len(reader.bios) + 1:] + "{" + option + "}" + str(suffix.value).strip('"') + return option, name + + +def boot_disks( + cg: Configgen, read: Callable[[str], str], specs: tuple[BootDisk, ...] = BOOT_DISKS +) -> list[Launcher]: + """`if system.config[o] in [...]: X = bios + o + ext else: X = bios + o + ext2`.""" + launchers = [] + for spec in specs: + src = cg.source(f"{cg.generators}/{spec.path}") + reader = _Reader(cg, src) + module = spec.path.removesuffix(".py").replace("/", ".") + emulator = ( + ", ".join(_caller_cores(cg, spec.caller)) if spec.caller else cg.emulator(module) + ) + found = 0 + for chain in ast.walk(src.tree): + if not ( + isinstance(chain, ast.If) and isinstance(chain.test, ast.Compare) + and isinstance(chain.test.ops[0], ast.In) + and isinstance(chain.test.comparators[0], ast.List) + and ast.unparse(chain.test.left).startswith("system.config[") + ): + continue + branches = [(chain.body, True), (chain.orelse, False)] + templates = [] + for body, listed in branches: + flag = next( + (str(s.value.value) for s in body if isinstance(s, ast.Assign) + and isinstance(s.value, ast.Constant) and str(s.value.value).startswith("-")), + None, + ) + named = [ + (s, t) for s in body if isinstance(s, ast.Assign) + and (t := _option_template(reader, s.value)) is not None + ] + if flag is None or len(named) != 1: + break + templates.append((listed, flag, *named[0])) + if len(templates) != 2: + continue + option = templates[0][3][0] + if option != templates[1][3][0] or ast.unparse(chain.test.left) != f"system.config[{option!r}]": + src.fail(f"line {chain.lineno} names a file from two options") + listed_values = [ + str(e.value) for e in chain.test.comparators[0].elts if isinstance(e, ast.Constant) + ] + offered, lines = option_choices(read, (*spec.feature, option)) + holder = reader.parents.get(id(chain)) + body = getattr(holder, "body", []) + index = next(i for i, stmt in enumerate(body) if stmt is chain) + added = body[index + 1] if index + 1 < len(body) else None + if not ( + isinstance(added, ast.AugAssign) and isinstance(added.value, ast.List) + and ast.unparse(added.target) in COMMAND_NAMES + ): + src.fail(f"the file chosen at line {chain.lineno} is not put on the command line") + for listed, flag, statement, (_, name) in templates: + values = listed_values if listed else [v for v in offered if v not in listed_values] + if not values: + src.fail(f"no {option} value reaches the branch at line {statement.lineno}") + tests, spans = reader.conditions(statement) + found += 1 + launchers.append(Launcher( + emulator=emulator, build=spec.build, kind="fixed_name", names=[name], + expands={option: values}, case_insensitive=False, search_dir=cg.bios, + passes_to=f"command line {flag}", fails_when_missing=False, + depends_on_option=f"{option} ({' and '.join(tests)})", + source_ref=", ".join([ + src.ref([*spans, src.span(statement), _lines(added)]), + f"{ES_FEATURES}:{lines[0]}-{lines[1]}" if not listed else "", + ]).rstrip(", "), + )) + if not found: + src.fail("names no BIOS file from an option value") + return launchers + + EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = ( *SHARED, lexaloffle, core_file_options, relative_folders, ) @@ -919,6 +1059,7 @@ def extract_launchers(read: Callable[[str], str]) -> list[dict[str, Any]]: cg = configgen(read) launchers = [launcher for extractor in EXTRACTORS for launcher in extractor(cg)] launchers += pico8_installer(read, cg.bios) + launchers += boot_disks(cg, read) for spec in GENERIC_SITES: launchers += bios_sites(cg, spec) return [launcher.entry() for launcher in launchers] diff --git a/tests/test_launchers_knulli.py b/tests/test_launchers_knulli.py index 9e235fbec..62062446a 100644 --- a/tests/test_launchers_knulli.py +++ b/tests/test_launchers_knulli.py @@ -60,6 +60,7 @@ IMPORTER = text( 'hatari': ('hatari.hatariGenerator', 'HatariGenerator'), 'melonds': ('melonds.melondsGenerator', 'MelonDSGenerator'), 'pcsx2': ('pcsx2.pcsx2Generator', 'Pcsx2Generator'), + 'mame': ('mame.mameGenerator', 'MameGenerator'), } """ ) @@ -329,6 +330,38 @@ PCSX2 = text( """ ) +# mameGenerator.py: the Macintosh boot disk named from the bootdisk option. +MAME = text( + """ + class MameGenerator(Generator): + def generate(self, system, rom, playersControllers, metadata, guns, wheels, gameResolution): + if system.name == "macintosh" and system.isOptSet("bootdisk"): + if system.config["bootdisk"] in [ "macos30", "macos608" ]: + bootType = "-flop1" + bootDisk = "/userdata/bios/" + system.config["bootdisk"] + ".img" + else: + bootType = "-hard" + bootDisk = "/userdata/bios/" + system.config["bootdisk"] + ".chd" + commandArray += [ bootType, bootDisk ] + """ +) + +FEATURES = text( + """ + mame: + systems: + macintosh: + cfeatures: + bootdisk: + prompt: BOOT DISK + choices: + "System 3.0 (LD Floppy)": macos3 + "System 6.0.8 (LD Floppy)": macos608 + "System 7.5.5 (Hard Drive)": mac755 + """ +) + + def tree(**overrides: str) -> dict[str, str]: files = { f"{kl.CONFIGGEN}/batoceraPaths.py": PATHS, @@ -347,6 +380,8 @@ def tree(**overrides: str) -> dict[str, str]: f"{G}/hatari/hatariGenerator.py": HATARI, f"{G}/melonds/melondsGenerator.py": MELONDS, f"{G}/pcsx2/pcsx2Generator.py": PCSX2, + f"{G}/mame/mameGenerator.py": MAME, + kl.ES_FEATURES: FEATURES, } files.update(overrides) return files @@ -550,6 +585,34 @@ class CopiesAndFolders(unittest.TestCase): ) +class BootDisks(unittest.TestCase): + def records(self, **overrides: str) -> list: + files = tree(**overrides) + return kl.boot_disks(kl.configgen(files.__getitem__), files.__getitem__, kl.BOOT_DISKS[:1]) + + def test_each_branch_is_a_template_over_the_values_that_reach_it(self): + floppy, hard = self.records() + self.assertEqual((floppy.names, floppy.passes_to), (["{bootdisk}.img"], "command line -flop1")) + self.assertEqual(floppy.expands, {"bootdisk": ["macos30", "macos608"]}) + # The values the menu offers that the code's list does not take. + self.assertEqual((hard.names, hard.passes_to), (["{bootdisk}.chd"], "command line -hard")) + self.assertEqual(hard.expands, {"bootdisk": ["macos3", "mac755"]}) + self.assertTrue(hard.depends_on_option.startswith("bootdisk (system.name == 'macintosh'")) + self.assertEqual( + hard.source_ref, + f"{G}/mame/mameGenerator.py:3-4, 9-10, {kl.ES_FEATURES}:8-10", + ) + + def test_a_name_built_another_way_raises(self): + broken = MAME.replace('"/userdata/bios/" + system.config["bootdisk"] + ".chd"', "disk") + with self.assertRaises(LauncherShapeError): + self.records(**{f"{G}/mame/mameGenerator.py": broken}) + + def test_an_option_the_menu_no_longer_offers_raises(self): + with self.assertRaises(LauncherShapeError): + self.records(**{kl.ES_FEATURES: FEATURES.replace("bootdisk:", "other:")}) + + class PlatformFile(unittest.TestCase): @classmethod def setUpClass(cls) -> None: From 25e515f982af3148801bb4eccbfcf6e7ee65cf86 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:03:08 +0100 Subject: [PATCH 7/7] docs: quote the -rp flag in amberelec_launchers --- scripts/scraper/amberelec_launchers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/scraper/amberelec_launchers.py b/scripts/scraper/amberelec_launchers.py index 725c8cced..259d46cd4 100644 --- a/scripts/scraper/amberelec_launchers.py +++ b/scripts/scraper/amberelec_launchers.py @@ -2,7 +2,7 @@ EmulationStation hands every launch to runemu.py. A libretro core runs in RetroArch or retrorun with the system directory their configs name, MAME -machines get a .cmd file with a -rp search path, and every other emulator +machines get a .cmd file with a `-rp` search path, and every other emulator runs the script its standalone_emulators table names (/usr/bin/.sh, installed from the package). Those scripts, and the configs their packages install, say which file under /storage/roms/bios is opened: yabasanshiro.sh @@ -127,7 +127,7 @@ def retrorun(read: Read) -> list[Launcher]: def mame(runemu: Runemu) -> list[Launcher]: - """The -rp search path of the .cmd file runemu writes for a MAME machine.""" + """The `-rp` search path of the .cmd file runemu writes for a MAME machine.""" src = runemu.src commands = [ node for node in ast.walk(src.function("get_retroarch_command"))