diff --git a/schemas/platform.schema.json b/schemas/platform.schema.json index 01cfc484..a2ab7805 100644 --- a/schemas/platform.schema.json +++ b/schemas/platform.schema.json @@ -212,6 +212,10 @@ "type": "string", "description": "Options or conditions in the launcher code that decide whether this record applies" }, + "device": { + "type": "string", + "description": "Devices whose shipped config this record reads, comma-separated, set only where the configs of a package differ between devices" + }, "source_ref": { "type": "string", "description": "path:lines in the platform's repository at the pinned revision" diff --git a/scripts/scraper/batocera_launchers.py b/scripts/scraper/batocera_launchers.py index e92422b2..05e16c3e 100644 --- a/scripts/scraper/batocera_launchers.py +++ b/scripts/scraper/batocera_launchers.py @@ -50,6 +50,7 @@ class Launcher: fails_when_missing: bool | str = False fails_scope: str | None = None depends_on_option: str | None = None + device: str | None = None def entry(self) -> dict[str, Any]: """The platform YAML form, in a fixed key order, unset keys left out.""" @@ -57,7 +58,7 @@ class Launcher: "emulator", "build", "kind", "slot", "names", "expands", "case_insensitive", "search_dir", "passes_to", "fails_when_missing", "fails_scope", "depends_on_option", - "source_ref", + "device", "source_ref", ) return {key: getattr(self, key) for key in order if getattr(self, key) is not None} diff --git a/scripts/scraper/rocknix_launchers.py b/scripts/scraper/rocknix_launchers.py new file mode 100644 index 00000000..a07ae601 --- /dev/null +++ b/scripts/scraper/rocknix_launchers.py @@ -0,0 +1,648 @@ +"""ROCKNIX's launcher layer, read from its packages at the pinned tag. + +ROCKNIX starts each standalone emulator from a shell script and installs a +config with the package. Between a file in /storage/roms/bios and the +emulator, the config names the path (melonDS, xemu, mednafen), the script +searches or links (yabasanshiro, drastic, dolphin), or the emulator is +handed a directory and left to choose (DuckStation, AetherSX2). Several +packages ship one config per device: a record carries `device` only where +the value read differs between them. +""" + +from __future__ import annotations + +import posixpath +import re +from collections.abc import Callable +from dataclasses import dataclass + +try: + from .batocera_launchers import Launcher + from .shell_launchers import ( + LauncherPatternMissing, + Link, + RepoTree, + Script, + Setting, + describe_keys, + settings, + source_ref, + ) +except ImportError: + from batocera_launchers import Launcher + from shell_launchers import ( + LauncherPatternMissing, + Link, + RepoTree, + Script, + Setting, + describe_keys, + settings, + source_ref, + ) + +ROCKNIX = "projects/ROCKNIX" +DEVICES = f"{ROCKNIX}/devices" +STANDALONE = f"{ROCKNIX}/packages/emulators/standalone" +RETROARCH = f"{ROCKNIX}/packages/emulators/libretro/retroarch" +BIOS_ROOT = "/storage/roms/bios" + +Lines = list[tuple[str, set[int]]] + + +@dataclass +class DeviceConfig: + """The config file one or more devices install, and the lines that pick it.""" + + path: str + devices: list[str] + mk_lines: set[int] + + +def device_configs( + tree: RepoTree, package: str, file: str, folder: str = "config" +) -> list[DeviceConfig]: + """Each distinct file package.mk installs as `file` for some device. + + The directory comes from `${PKG_DIR}//${DEVICE}/` or from a + `case ${DEVICE}` naming it. A device whose directory is absent does not + get the package. A directory that is a symlink resolves to its target. + """ + mk_path = f"{package}/package.mk" + mk = Script(mk_path, tree.read(mk_path)) + arms: list[tuple[list[str], str]] = [] + mk_lines: set[int] = set() + for block in re.finditer(r'case\s+"?\$\{?DEVICE\}?"?\s+in(.*?)\besac\b', mk.text, re.S): + body = block.group(1) + if not re.search(rf"CONFIG=|\$\{{PKG_DIR\}}/{folder}/", body): + continue + offset = block.start(1) + for arm in re.finditer(r"^\s*([\w*|]+)\)(.*?);;", body, re.M | re.S): + chosen = re.search( + rf'CONFIG="([^"]+)"|\$\{{PKG_DIR\}}/{folder}/([^/\s]+)', arm.group(2) + ) + if chosen: + arms.append((arm.group(1).split("|"), chosen.group(1) or chosen.group(2))) + mk_lines.add(mk.text.count("\n", 0, offset + arm.start(2)) + 1) + break + if not arms: + number, _ = mk.find( + rf"\$\{{PKG_DIR\}}/{folder}/\$\{{DEVICE\}}/", f"{folder}/${{DEVICE}} copy" + ) + arms, mk_lines = [(["*"], "${DEVICE}")], {number} + root = f"{package}/{folder}" + entries = tree.listdir(root) + by_path: dict[str, DeviceConfig] = {} + for device, entry in tree.listdir(DEVICES).items(): + if entry.kind != "dir": + continue + arm = next((d for patterns, d in arms if device in patterns or "*" in patterns), None) + if arm is None: + continue + name = device if arm in ("${DEVICE}", "${CONFIG}") else arm + found = entries.get(name) + if found is None: + continue + if found.kind == "link": + name = found.target.rstrip("/") + path = f"{root}/{name}/{file}" + by_path.setdefault(path, DeviceConfig(path, [], mk_lines)).devices.append(device) + if not by_path: + raise LauncherPatternMissing(root, f"{file} for any device") + return sorted(by_path.values(), key=lambda c: c.path) + + +def per_device( + tree: RepoTree, + configs: list[DeviceConfig], + mk_path: str, + read: Callable[[str, str], tuple[object, set[int]]], +) -> list[tuple[str | None, object, Lines]]: + """Configs grouped by what read() finds in them. + + read(path, text) returns a comparable value and the lines it comes from. + One group gives one result without a device. Otherwise each group names + its devices, in the order of the devices directory. + """ + groups: dict[object, tuple[list[str], Lines]] = {} + for config in configs: + value, lines = read(config.path, tree.read(config.path)) + devices, refs = groups.setdefault(value, ([], [])) + devices.extend(config.devices) + refs.append((config.path, lines)) + mk_lines = set().union(*(c.mk_lines for c in configs)) + if len(groups) == 1: + ((value, (_, refs)),) = groups.items() + return [(None, value, [*refs, (mk_path, mk_lines)])] + order = list(tree.listdir(DEVICES)) + return [ + (", ".join(sorted(devices, key=order.index)), value, [*refs, (mk_path, mk_lines)]) + for value, (devices, refs) in sorted( + groups.items(), key=lambda g: min(order.index(d) for d in g[1][0]) + ) + ] + + +def _file_under(value: str, root: str) -> bool: + return value.startswith(root.rstrip("/") + "/") + + +@dataclass(frozen=True) +class FixedPaths: + """Keys of a config that name a file under one directory.""" + + keys: tuple[tuple[str, str], ...] + names: tuple[str, ...] + search_dir: str + + +def fixed_paths( + found: list[Setting], keys: list[tuple[str, str]], path: str +) -> tuple[FixedPaths, set[int]]: + """The values of keys, as names relative to the directory they share.""" + by_key = {(s.section, s.key): s for s in found} + missing = [k for k in keys if k not in by_key] + if missing: + raise LauncherPatternMissing(path, f"key {missing[0][1]}") + values = [by_key[k].value for k in keys if by_key[k].value] + lines = {by_key[k].line for k in keys} + if not values: + return FixedPaths(tuple(keys), (), ""), lines + folder = posixpath.commonpath([posixpath.dirname(v) for v in values]) + names = tuple(posixpath.relpath(v, folder) for v in values) + return FixedPaths(tuple(keys), names, folder), lines + + +def bios_keys(tree: RepoTree, paths: list[str], root: str = BIOS_ROOT) -> list[tuple[str, str]]: + """Keys whose value is a file under root in at least one of the configs.""" + keys: list[tuple[str, str]] = [] + for path in paths: + for setting in settings(tree.read(path)): + key = (setting.section, setting.key) + if _file_under(setting.value, root) and key not in keys: + keys.append(key) + if not keys: + raise LauncherPatternMissing(paths[0], f"key naming a file under {root}") + return keys + + +def config_records( + tree: RepoTree, + emulator: str, + package: str, + file: str, + passes_to_suffix: str = "", + **fields: object, +) -> list[Launcher]: + """fixed_name records for the bios paths of a config installed per device.""" + configs = device_configs(tree, package, file) + keys = bios_keys(tree, [c.path for c in configs]) + name = posixpath.basename(file) + + def read(path: str, text: str) -> tuple[object, set[int]]: + return fixed_paths(settings(text), keys, path) + + records = [] + for device, value, refs in per_device(tree, configs, f"{package}/package.mk", read): + assert isinstance(value, FixedPaths) + described = describe_keys(name, value.keys) + records.append( + Launcher( + emulator=emulator, + build="standalone", + kind="fixed_name", + names=list(value.names), + source_ref=source_ref(refs), + search_dir=value.search_dir or None, + passes_to=( + (described + passes_to_suffix) if value.names else f"{described} left empty" + ), + device=device, + **fields, + ) + ) + return records + + +def retroarch(tree: RepoTree) -> list[Launcher]: + """system_directory of the retroarch.cfg each device installs.""" + configs = device_configs(tree, RETROARCH, "retroarch.cfg", folder="sources") + wanted = ("system_directory", "systemfiles_in_content_dir") + + def read(path: str, text: str) -> tuple[object, set[int]]: + found = {s.key: s for s in settings(text) if s.key in wanted} + if "system_directory" not in found: + raise LauncherPatternMissing(path, "system_directory") + return tuple((k, found[k].value) for k in wanted if k in found), { + s.line for s in found.values() + } + + records = [] + for device, value, refs in per_device(tree, configs, f"{RETROARCH}/package.mk", read): + assert isinstance(value, tuple) + values = dict(value) + records.append( + Launcher( + emulator="retroarch", + build="libretro", + kind="passes_path", + names=[], + source_ref=source_ref(refs), + search_dir=values["system_directory"], + passes_to="retroarch.cfg " + ", ".join(f'{k} = "{v}"' for k, v in value), + device=device, + ) + ) + return records + + +def amiberry(tree: RepoTree) -> list[Launcher]: + """The A500 or A1200 config, chosen by the game name, then the kickstarts link.""" + package = f"{STANDALONE}/amiberry" + script = Script( + f"{package}/scripts/start_amiberry.sh", tree.read(f"{package}/scripts/start_amiberry.sh") + ) + test_line, test = script.find(r"grep -E '([^']+)'", "machine choice by game name") + then_line, then = script.find( + r"cp \S+/(\w+-default\.uae)", "config copied for a match", test_line + ) + else_line, _ = script.find(r"^\s*else\b", "else of the machine choice", then_line) + other_line, other = script.find( + r"cp \S+/(\w+-default\.uae)", "config copied otherwise", else_line + ) + sed_line, sed = script.find( + r'sed -i "s#(@\w+@)#\$\{(\w+)\}#g"', "placeholder substitution", other_line + ) + used: set[int] = {test_line, then_line, else_line, other_line, sed_line} + root = script.expand("${" + sed.group(2) + "}", sed_line, used) + records = [] + for config, condition in ( + (then.group(1), f"game name matches {test.group(1)}"), + (other.group(1), f"game name does not match {test.group(1)}"), + ): + path = f"{package}/config/conf/{config}" + found = [ + Setting(s.section, s.key, s.value.replace(sed.group(1), root), s.line) + for s in settings(tree.read(path)) + ] + keys = [(s.section, s.key) for s in found if _file_under(s.value, root)] + if not keys: + raise LauncherPatternMissing(path, f"key naming a file under {sed.group(1)}") + value, lines = fixed_paths(found, keys, path) + slot = re.fullmatch(r"Amiga(\w+)-default\.uae", config) + records.append( + Launcher( + emulator="amiberry", + build="standalone", + kind="fixed_name", + names=list(value.names), + source_ref=source_ref([(script.path, used), (path, lines)]), + slot=slot.group(1) if slot else config, + search_dir=value.search_dir, + passes_to=f"{describe_keys(config, value.keys)} (amiberry -f)", + depends_on_option=condition, + ) + ) + mk = Script(f"{package}/package.mk", tree.read(f"{package}/package.mk")) + links = mk.links(BIOS_ROOT, fixed={"INSTALL": ""}) + if not links: + raise mk.missing(f"ln -s of {BIOS_ROOT}") + records.extend(link_record("amiberry", mk, link) for link in links) + return records + + +def link_record(emulator: str, script: Script, link: Link, **fields: object) -> Launcher: + return Launcher( + emulator=emulator, + build="standalone", + kind="links", + names=list(link.names), + source_ref=source_ref([(script.path, link.lines)]), + expands=link.expands or None, + search_dir=link.bios_side, + passes_to=f"symlink {link.link} -> {link.target}", + **fields, + ) + + +def hatari(tree: RepoTree) -> list[Launcher]: + package = f"{STANDALONE}/hatarisa" + script = Script( + f"{package}/scripts/start_hatari.sh", tree.read(f"{package}/scripts/start_hatari.sh") + ) + copy_line, copy = script.find(r"cp \S+/([\w-]+\.cfg) ", "default config copy") + run_line, _ = script.find(r"--configfile", "--configfile", copy_line) + path = f"{package}/config/conf/{copy.group(1)}" + found = settings(tree.read(path)) + keys = [(s.section, s.key) for s in found if _file_under(s.value, BIOS_ROOT)] + if not keys: + raise LauncherPatternMissing(path, f"key naming a file under {BIOS_ROOT}") + value, lines = fixed_paths(found, keys, path) + return [ + Launcher( + emulator="hatari", + build="standalone", + kind="fixed_name", + names=list(value.names), + source_ref=source_ref([(path, lines), (script.path, {copy_line, run_line})]), + search_dir=value.search_dir, + passes_to=f"{describe_keys(copy.group(1), value.keys)} (--configfile)", + ) + ] + + +def melonds(tree: RepoTree) -> list[Launcher]: + package = f"{STANDALONE}/melonds-sa" + script = Script( + f"{package}/scripts/start_melonds.sh", tree.read(f"{package}/scripts/start_melonds.sh") + ) + enable_line, _ = script.find(r"ExternalBIOSEnable=1", "ExternalBIOSEnable=1") + tests = [ + (number, found) + for number, line in script.code() + if number < enable_line and (found := re.search(r'if \[ "\$(\w+)" = "(\w+)" \]', line)) + ] + if not tests: + raise script.missing("test that enables the external BIOS") + test_line, test = tests[-1] + setting_line, setting = script.find( + rf"^\s*{test.group(1)}=\$\(get_setting (\w+) ", f"setting read into {test.group(1)}" + ) + condition = f"{setting.group(1)} = {test.group(2)} sets ExternalBIOSEnable=1, otherwise 0" + records = config_records(tree, "melonds", package, "melonDS.ini", depends_on_option=condition) + for record in records: + record.source_ref += ( + f", {source_ref([(script.path, {setting_line, test_line, enable_line})])}" + ) + return records + + +def dsperate(tree: RepoTree) -> list[Launcher]: + package = f"{STANDALONE}/dsperate-sa" + script = Script( + f"{package}/scripts/start_dsperate.sh", tree.read(f"{package}/scripts/start_dsperate.sh") + ) + run_line, _ = script.find(r"/usr/bin/dsperate --config ", "dsperate --config") + records = config_records(tree, "dsperate", package, "dsperate.ini", " (--config)") + for record in records: + record.source_ref += f", {source_ref([(script.path, {run_line})])}" + return records + + +def xemu(tree: RepoTree) -> list[Launcher]: + package = f"{STANDALONE}/xemu-sa" + script = Script( + f"{package}/scripts/start_xemu.sh", tree.read(f"{package}/scripts/start_xemu.sh") + ) + copy_line, _ = script.find( + r'cp -r "\$\{SOURCE_DIR\}/\$\{XEMU_INI\}"', "xemu.toml copy at launch" + ) + records = config_records(tree, "xemu", package, "xemu.toml", " (copied at each launch)") + for record in records: + record.source_ref += f", {source_ref([(script.path, {copy_line})])}" + return records + + +def nanoboyadvance(tree: RepoTree) -> list[Launcher]: + package = f"{STANDALONE}/nanoboyadvance-sa" + path = f"{package}/config/config.toml" + found = settings(tree.read(path)) + keys = [(s.section, s.key) for s in found if _file_under(s.value, BIOS_ROOT)] + if not keys: + raise LauncherPatternMissing(path, f"key naming a file under {BIOS_ROOT}") + value, lines = fixed_paths(found, keys, path) + mk_path = f"{package}/package.mk" + mk_line, _ = Script(mk_path, tree.read(mk_path)).find( + r"config/config\.toml", "config.toml install" + ) + return [ + Launcher( + emulator="nanoboyadvance", + build="standalone", + kind="fixed_name", + names=list(value.names), + source_ref=source_ref([(path, lines), (mk_path, {mk_line})]), + search_dir=value.search_dir, + passes_to=describe_keys("config.toml", value.keys), + ) + ] + + +def mednafen(tree: RepoTree) -> list[Launcher]: + """The firmware keys of mednafen.template, one record per module.""" + package = f"{STANDALONE}/mednafen" + path = f"{package}/config/common/mednafen.template" + found = settings(tree.read(path), separator=" ") + folder = next((s for s in found if s.key == "filesys.path_firmware"), None) + if folder is None: + raise LauncherPatternMissing(path, "filesys.path_firmware") + pattern = re.compile(r"(\w+)\.(\w*bios\w*|ggrom|cart\.\w+_path)") + firmware = [s for s in found if pattern.fullmatch(s.key) and s.value and not s.value.isdigit()] + if not firmware: + raise LauncherPatternMissing(path, "firmware keys") + script_path = f"{package}/scripts/start_mednafen.sh" + script = Script(script_path, tree.read(script_path)) + gen_line, _ = script.find(r"mednafen_gen_config\.sh", "config generation") + test_line, _ = script.find( + r'if \[ ! -f "\$MEDNAFEN_HOME/mednafen\.cfg" \]', "mednafen.cfg test" + ) + modules: dict[str, list[Setting]] = {} + for setting in firmware: + modules.setdefault(setting.key.split(".", 1)[0], []).append(setting) + return [ + Launcher( + emulator="mednafen", + build="standalone", + kind="fixed_name", + names=list(dict.fromkeys(s.value for s in group)), + source_ref=source_ref( + [ + (path, {folder.line, *(s.line for s in group)}), + (script_path, {test_line, gen_line}), + ] + ), + slot=module, + search_dir=folder.value, + passes_to="mednafen.cfg " + ", ".join(s.key for s in group), + ) + for module, group in modules.items() + ] + + +def yabasanshiro(tree: RepoTree) -> list[Launcher]: + package = f"{STANDALONE}/yabasanshiro-sa" + script = Script( + f"{package}/scripts/start_yabasanshiro.sh", + tree.read(f"{package}/scripts/start_yabasanshiro.sh"), + ) + option_line, option = script.find(r"=\$\(get_setting (\w+) ", "BIOS option") + test_line, test = script.find(r'if \[ "\$\w+" = (\w+) \]', "BIOS option test", option_line) + loop = next((lp for lp in script.loops() if lp.start > test_line), None) + if loop is None: + raise script.missing("loop over BIOS names") + find_line, find = script.find( + rf"\$\(find (\S+) -name \$\{{{loop.var}\}}", "find of each name", loop.start, loop.end + ) + flag_line, flag = script.find( + r'="(-\w) \$\{\w+\}"', "flag the found path is passed with", find_line, loop.end + ) + break_line, _ = script.find(r"^\s*break\b", "break on the first found", flag_line, loop.end) + return [ + Launcher( + emulator="yabasanshiro", + build="standalone", + kind="names_searched", + names=loop.items, + source_ref=source_ref( + [(script.path, {option_line, test_line, *range(loop.start, loop.end + 1)})] + ), + case_insensitive=False, + search_dir=f"{find.group(1)} (recursive)", + passes_to=f"yabasanshiro {flag.group(1)} ", + fails_when_missing=False, + depends_on_option=f"{option.group(1)} = {test.group(1)} (game, then platform)", + ) + ] + + +def directory_records( + tree: RepoTree, + emulator: str, + package: str, + file: str, + folder_key: tuple[str, str], + file_keys: tuple[tuple[str, str], ...], +) -> list[Launcher]: + """scans_dir records: the config hands a directory and leaves the file keys empty.""" + configs = device_configs(tree, package, file) + name = posixpath.basename(file) + + def read(path: str, text: str) -> tuple[object, set[int]]: + found = {(s.section, s.key): s for s in settings(text)} + if folder_key not in found: + raise LauncherPatternMissing(path, f"[{folder_key[0]}] {folder_key[1]}") + set_files = [k for k in file_keys if k in found and found[k].value] + if set_files: + raise LauncherPatternMissing(path, f"empty [{set_files[0][0]}] {set_files[0][1]}") + lines = {found[k].line for k in (folder_key, *file_keys) if k in found} + return found[folder_key].value, lines + + unset = describe_keys("", file_keys).strip() + records = [] + for device, folder, refs in per_device(tree, configs, f"{package}/package.mk", read): + passes = f"{name} [{folder_key[0]}] {folder_key[1]} = {folder}, {unset} unset" + records.append( + Launcher( + emulator=emulator, + build="standalone", + kind="scans_dir", + names=[], + source_ref=source_ref(refs), + search_dir=folder, + passes_to=passes, + device=device, + ) + ) + return records + + +def duckstation(tree: RepoTree) -> list[Launcher]: + return directory_records( + tree, + "duckstation", + f"{STANDALONE}/duckstation-sa", + "settings.ini", + ("BIOS", "SearchDirectory"), + (("BIOS", "PathNTSCU"), ("BIOS", "PathNTSCJ"), ("BIOS", "PathPAL")), + ) + + +def aethersx2(tree: RepoTree) -> list[Launcher]: + return directory_records( + tree, + "aethersx2", + f"{STANDALONE}/aethersx2-sa", + "aethersx2/inis/PCSX2.ini", + ("Folders", "Bios"), + (("Filenames", "BIOS"),), + ) + + +def armsx2(tree: RepoTree) -> list[Launcher]: + return directory_records( + tree, + "armsx2", + f"{STANDALONE}/armsx2-sa", + "ARMSX2/inis/PCSX2.ini", + ("Folders", "Bios"), + (("Filenames", "BIOS"),), + ) + + +# Start scripts whose ln -s commands put a BIOS path where the emulator reads. +LINK_SCRIPTS = ( + ("drastic", f"{STANDALONE}/drastic-sa/scripts/start_drastic.sh"), + ("dolphin", f"{STANDALONE}/dolphin-sa/scripts/start_dolphin_gc.sh"), + ("flycast", f"{STANDALONE}/flycast-sa/scripts/start_flycast.sh"), + ("cemu", f"{STANDALONE}/cemu-sa/scripts/start_cemu.sh"), + ("rpcs3", f"{STANDALONE}/rpcs3-sa/scripts/start_rpcs3.sh"), +) + + +def links(tree: RepoTree) -> list[Launcher]: + records = [] + for emulator, path in LINK_SCRIPTS: + script = Script(path, tree.read(path)) + found = script.links(BIOS_ROOT) + if not found: + raise script.missing(f"ln -s of a path under {BIOS_ROOT}") + for link in found: + guard = _existence_guard(script, link) + if guard is not None: + link.lines.add(guard) + records.append( + link_record( + emulator, + script, + link, + case_insensitive=False if guard is not None else None, + fails_when_missing=False, + ) + ) + return records + + +def _existence_guard(script: Script, link: Link) -> int | None: + """The line of a -e test of the BIOS-side path the link waits for, if any.""" + last = max(link.lines) + tests = [ + number + for number, line in script.code() + if number < last + and re.search( + r"\[ -e \S*" + re.escape(link.bios_side) + "/", script.expand(line, number, set()) + ) + ] + return tests[-1] if tests and last - tests[-1] <= 3 else None + + +EXTRACTORS: tuple[Callable[[RepoTree], list[Launcher]], ...] = ( + retroarch, + amiberry, + hatari, + melonds, + dsperate, + nanoboyadvance, + xemu, + mednafen, + yabasanshiro, + duckstation, + aethersx2, + armsx2, + links, +) + + +def extract_launchers(tree: RepoTree) -> list[dict]: + """Every launcher record the extractors find, in extractor order.""" + return [record.entry() for extractor in EXTRACTORS for record in extractor(tree)] diff --git a/scripts/scraper/rocknix_scraper.py b/scripts/scraper/rocknix_scraper.py index fddc022e..58719dd1 100644 --- a/scripts/scraper/rocknix_scraper.py +++ b/scripts/scraper/rocknix_scraper.py @@ -24,6 +24,8 @@ try: fetch_github_latest_version, requirement_entry, ) + from .rocknix_launchers import extract_launchers + from .shell_launchers import RepoTree except ImportError: from base_scraper import ( BaseScraper, @@ -31,6 +33,8 @@ except ImportError: fetch_github_latest_version, requirement_entry, ) + from rocknix_launchers import extract_launchers + from shell_launchers import RepoTree PLATFORM_NAME = "rocknix" @@ -53,6 +57,14 @@ STANDALONE_TREE_URL = ( ) +def launcher_revision(url: str) -> str: + """The tag or commit a rocknix-systems URL is read at.""" + found = re.search(r"/ROCKNIX/distribution/([^/]+)/projects/", url) + if found is None: + raise ValueError(f"{url} names no ROCKNIX/distribution revision") + return found.group(1) + + def standalone_emulators(package_names: list[str]) -> list[str]: """The standalone emulators ROCKNIX packages, by the names profiles use. @@ -274,6 +286,10 @@ class Scraper(BaseScraper): # emulator as a RetroArch core. "dual_mode_cores": dual_mode_emulators(standalone, self._fetch_emulators_mk()), "systems": systems, + # Read at the revision the BIOS list comes from. + "launchers": extract_launchers( + RepoTree.github(GITHUB_REPO, launcher_revision(self.url)) + ), } diff --git a/scripts/scraper/shell_launchers.py b/scripts/scraper/shell_launchers.py new file mode 100644 index 00000000..35177325 --- /dev/null +++ b/scripts/scraper/shell_launchers.py @@ -0,0 +1,531 @@ +"""Readers for launchers written as shell scripts and key=value configs. + +ROCKNIX and EmuDeck launch emulators from bash: a start script links or +writes a BIOS path, a config shipped with the package names the file. These +helpers read that text without running it. Each one targets a construct +(an assignment, a for loop, an ln command, a key of a config) and the +extractors built on them raise LauncherPatternMissing when the construct is +gone, so a rewrite upstream stops the scrape instead of publishing nothing. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Callable, Iterable +from dataclasses import dataclass, field + +try: + from .base_scraper import fetch_bytes, github_headers + from .batocera_launchers import LauncherShapeError +except ImportError: + from base_scraper import fetch_bytes, github_headers + from batocera_launchers import LauncherShapeError + +NAME = r"[A-Za-z_][A-Za-z0-9_]*" +# A single-quoted "$" is literal: it is carried as this mark through expansion. +LITERAL_DOLLAR = "\x00" +# A loop variable expands to this mark around its name, so the caller can +# tell which part of a path the loop supplies. +LOOP_MARK = "\x01" + + +class LauncherPatternMissing(LauncherShapeError): + """A construct an extractor reads is absent from the launcher source.""" + + def __init__(self, path: str, what: str) -> None: + super().__init__(f"{path}: {what} not found") + + +def span(first: int, last: int) -> str: + return str(first) if first == last else f"{first}-{last}" + + +def merge_lines(lines: Iterable[int]) -> list[tuple[int, int]]: + """Line numbers as sorted spans, a gap of one line joined.""" + spans: list[tuple[int, int]] = [] + for line in sorted(set(lines)): + if spans and line <= spans[-1][1] + 2: + spans[-1] = (spans[-1][0], line) + else: + spans.append((line, line)) + return spans + + +def source_ref(parts: Iterable[tuple[str, Iterable[int]]]) -> str: + """path:a-b, c for each file, files in the order given.""" + merged: dict[str, set[int]] = {} + for path, lines in parts: + merged.setdefault(path, set()).update(lines) + return ", ".join( + f"{path}:{', '.join(span(*s) for s in merge_lines(lines))}" + for path, lines in merged.items() + ) + + +@dataclass +class Word: + """A shell word: as written, and with quotes removed.""" + + raw: str + text: str + + +def words(line: str) -> list[Word]: + """The words of one command line, quotes removed, comments dropped. + + Variables are left unexpanded, each written ${NAME} so the quoting that + ended a name ("$a""b") still ends it. A "$" inside single quotes is kept + as LITERAL_DOLLAR so a later expansion leaves it alone. + """ + result: list[Word] = [] + raw, text, quote, started, segment = "", "", "", False, 0 + + def close_segment() -> None: + nonlocal text, segment + text = text[:segment] + re.sub(rf"\$({NAME})", r"${\1}", text[segment:]) + segment = len(text) + + i = 0 + while i < len(line): + char = line[i] + if quote == "'": + raw += char + if char == "'": + quote = "" + segment = len(text) + else: + text += LITERAL_DOLLAR if char == "$" else char + elif quote == '"': + raw += char + if char == "\\" and i + 1 < len(line) and line[i + 1] in '"\\$`': + raw += line[i + 1] + text += LITERAL_DOLLAR if line[i + 1] == "$" else line[i + 1] + i += 1 + elif char == '"': + quote = "" + close_segment() + else: + text += char + elif char in "'\"": + close_segment() + quote, started = char, True + raw += char + elif char == "\\" and i + 1 < len(line): + raw += char + line[i + 1] + text += line[i + 1] + started = True + i += 1 + elif char == "#" and not started: + break + elif char.isspace() or char == ";": + if started: + close_segment() + result.append(Word(raw, text)) + raw, text, started, segment = "", "", False, 0 + if char == ";": + result.append(Word(";", ";")) + else: + raw += char + text += char + started = True + i += 1 + if started: + close_segment() + result.append(Word(raw, text)) + return result + + +@dataclass +class Assignment: + line: int + value: str + + +@dataclass +class Loop: + """A for loop: its variable, its items and the lines they come from.""" + + var: str + items: list[str] + start: int + end: int + lines: set[int] = field(default_factory=set) + + +@dataclass +class Link: + """An ln -s whose link or target lies under the BIOS root.""" + + target: str + link: str + bios_side: str + names: list[str] + lines: set[int] + bios_is_target: bool = True + expands: dict[str, list[str]] = field(default_factory=dict) + + +def _placeholders(text: str, marks: dict[str, Loop]) -> str: + """Loop marks written as {VAR}, the placeholder `expands` resolves.""" + for mark, loop in marks.items(): + text = text.replace(mark, "{" + loop.var + "}") + return text + + +class Script: + """One shell script or makefile, read line by line.""" + + def __init__(self, path: str, text: str) -> None: + self.path = path + self.text = text + self.lines = text.splitlines() + self._assignments: dict[str, list[Assignment]] = {} + self._arrays: dict[str, tuple[int, list[str]]] = {} + for number, line in enumerate(self.lines, 1): + found = re.match(rf"\s*(?:local\s+|export\s+)?({NAME})=(.*)$", line) + if found is None: + continue + array = re.fullmatch(r"\((.*)\)\s*", found.group(2)) + if array: + self._arrays[found.group(1)] = (number, [w.text for w in words(array.group(1))]) + continue + value = words(found.group(2)) + if value and value[0].raw: + self._assignments.setdefault(found.group(1), []).append( + Assignment(number, value[0].text) + ) + + def missing(self, what: str) -> LauncherPatternMissing: + return LauncherPatternMissing(self.path, what) + + def code(self) -> Iterable[tuple[int, str]]: + """Lines that are not comments.""" + for number, line in enumerate(self.lines, 1): + if not line.lstrip().startswith("#"): + yield number, line + + def find( + self, pattern: str, what: str, start: int = 1, end: int | None = None + ) -> tuple[int, re.Match[str]]: + """The first code line from start to end matching pattern.""" + for number, line in self.code(): + if number < start or (end is not None and number > end): + continue + found = re.search(pattern, line) + if found: + return number, found + raise self.missing(what) + + def assignment_ending(self, suffix: str) -> Assignment | None: + """The first assignment whose value ends with suffix.""" + return next( + (a for found in self._assignments.values() for a in found if a.value.endswith(suffix)), + None, + ) + + def value(self, name: str, before: int) -> Assignment | None: + """The last assignment of name above a line, else the first below it.""" + assignments = self._assignments.get(name, []) + above = [a for a in assignments if a.line < before] + if above: + return above[-1] + return assignments[0] if assignments else None + + def expand( + self, text: str, line: int, used: set[int], fixed: dict[str, str] | None = None + ) -> str: + """Variables replaced by their assigned value, unknown ones as ${NAME}.""" + fixed = fixed or {} + + def replace(found: re.Match[str], depth: int) -> str: + name = found.group(1) or found.group(2) + if name in fixed: + return fixed[name] + assignment = self.value(name, line) + if assignment is None or depth > 8: + return "${" + name + "}" + used.add(assignment.line) + return expand_once(assignment.value, assignment.line, depth + 1) + + def expand_once(value: str, at: int, depth: int) -> str: + nonlocal line + saved, line = line, at + try: + return re.sub(rf"\$\{{({NAME})\}}|\$({NAME})", lambda m: replace(m, depth), value) + finally: + line = saved + + return expand_once(text, line, 0).replace(LITERAL_DOLLAR, "$") + + def loops(self) -> list[Loop]: + """Every for loop, items from a word list or from "${ARRAY[@]}".""" + found_loops = [] + for number, line in self.code(): + head = re.match(rf"\s*for ({NAME}) in (.*?)\s*(?:;\s*do)?\s*$", line) + if head is None: + continue + items = [w.text for w in words(head.group(2)) if w.text != ";"] + lines = {number} + if len(items) == 1 and (array := re.fullmatch(rf"\$\{{({NAME})\[@\]\}}", items[0])): + if array.group(1) not in self._arrays: + raise self.missing(f"array {array.group(1)}") + array_line, items = self._arrays[array.group(1)] + lines.add(array_line) + found_loops.append(Loop(head.group(1), items, number, self._loop_end(number), lines)) + return found_loops + + def _loop_end(self, start: int) -> int: + depth = 0 + for number, line in self.code(): + if number < start: + continue + if re.match(r"\s*(for|while|until)\b", line): + depth += 1 + if re.match(r"\s*done\b", line) or re.search(r";\s*done\s*$", line): + depth -= 1 + if depth == 0: + return number + raise self.missing(f"done of the loop at line {start}") + + def links(self, bios_root: str, fixed: dict[str, str] | None = None) -> list[Link]: + """ln -s commands with the link or its target under bios_root. + + Links in one loop, or written one by one as P/n linked to Q/n, are + one Link listing each n. + """ + loops = self.loops() + found: list[Link] = [] + for number, line in self.code(): + command = words(line) + heads = [ + i + for i, w in enumerate(command) + if w.raw == "ln" + and (i == 0 or command[i - 1].raw in {";", "then", "do", "&&", "||"}) + ] + for head in heads: + args = [] + for word in command[head + 1 :]: + if word.raw in {";", "&&", "||"}: + break + args.append(word) + options = [w.text for w in args if w.text.startswith("-")] + operands = [w.text for w in args if not w.text.startswith("-")] + if not any("s" in o for o in options) or len(operands) != 2: + continue + enclosing = [lp for lp in loops if lp.start < number <= lp.end] + env = dict(fixed or {}) + marks = {f"{LOOP_MARK}{lp.var}{LOOP_MARK}": lp for lp in enclosing} + env.update({lp.var: mark for mark, lp in marks.items()}) + used = {number} + target, link = (self.expand(op, number, used, env) for op in operands) + if not (_under(target, bios_root) or _under(link, bios_root)): + continue + bios_is_target = _under(target, bios_root) + bios_side = (target if bios_is_target else link).rstrip("/") + folder, _, last = bios_side.rpartition("/") + present = [m for m in marks if m in target or m in link] + if any(m in folder for m in present): + raise self.missing( + f"loop variable only in the last segment of the ln at line {number}" + ) + names: list[str] = [] + expands: dict[str, list[str]] = {} + if present: + bios_side = folder + if last in marks: + names = list(marks[last].items) + else: + names = [_placeholders(last, marks)] + expands = {lp.var: list(lp.items) for m, lp in marks.items() if m in last} + if bios_is_target and last not in link: + link = f"{link.rstrip('/')}/{last}" + for mark in present: + used |= marks[mark].lines + template = _placeholders(last, marks) if expands else "" + target, link = ( + p.replace(last, template) if last in p else _placeholders(p, marks) + for p in (target, link) + ) + duplicate = next((f for f in found if (f.target, f.link) == (target, link)), None) + if duplicate is not None: + duplicate.lines |= used + continue + found.append(Link(target, link, bios_side, names, used, bios_is_target, expands)) + return _merge_links(found) + + +def _under(path: str, root: str) -> bool: + return path.rstrip("/") == root.rstrip("/") or path.startswith(root.rstrip("/") + "/") + + +def _merge_links(links: list[Link]) -> list[Link]: + """Single links P/n -> Q/n sharing P and Q folded into one, names in order.""" + merged: list[Link] = [] + folds: dict[tuple[str, str, bool], tuple[Link, list[Link]]] = {} + for link in links: + bios = link.bios_side.rstrip("/") + bios_is_target = link.bios_is_target + other = (link.link if bios_is_target else link.target).rstrip("/") + name = bios.rsplit("/", 1)[-1] + if link.names or other.rsplit("/", 1)[-1] != name: + merged.append(link) + continue + key = (bios.rsplit("/", 1)[0], other.rsplit("/", 1)[0], bios_is_target) + if key in folds: + folds[key][1].append(link) + else: + folds[key] = (link, [link]) + merged.append(link) + folded: set[int] = set() + for (parent, other_parent, bios_is_target), (first, members) in folds.items(): + if len(members) == 1: + continue + first.names = [m.bios_side.rstrip("/").rsplit("/", 1)[-1] for m in members] + first.bios_side = parent + bios_end, other_end = f"{parent}/", f"{other_parent}/" + first.target, first.link = ( + (bios_end, other_end) if bios_is_target else (other_end, bios_end) + ) + for member in members[1:]: + first.lines |= member.lines + folded.add(id(member)) + return [link for link in merged if id(link) not in folded] + + +@dataclass +class Setting: + section: str + key: str + value: str + line: int + + +def settings(text: str, separator: str = "=") -> list[Setting]: + """key = value lines under their [section], quotes around the value removed. + + separator " " reads mednafen's "key value" form. + """ + found, section = [], "" + for number, line in enumerate(text.splitlines(), 1): + stripped = line.strip() + if not stripped or stripped[0] in "#;": + continue + header = re.fullmatch(r"\[([^\]]+)\]", stripped) + if header: + section = header.group(1) + continue + if separator == " ": + pair = re.fullmatch(r"(\S+)(?:\s+(.*))?", stripped) + else: + pair = re.fullmatch(r"([^=\s]+)\s*=\s*(.*)", stripped) + if pair is None: + continue + value = (pair.group(2) or "").strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + value = value[1:-1] + found.append(Setting(section, pair.group(1), value, number)) + return found + + +def describe_keys(file: str, keys: tuple[tuple[str, str], ...]) -> str: + """file [section] key, key [other] key, sections named where they change.""" + described, section = file, None + for index, (key_section, key) in enumerate(keys): + if key_section and key_section != section: + described += f" [{key_section}] {key}" + section = key_section + else: + described += (" " if index == 0 else ", ") + key + return described + + +@dataclass +class Entry: + """A directory entry: kind is "dir", "file" or "link".""" + + kind: str + target: str = "" + + +class RepoTree: + """Files of one repository revision, read on demand and kept.""" + + def __init__( + self, + read: Callable[[str], str], + listdir: Callable[[str], dict[str, Entry]], + ) -> None: + self._read = read + self._listdir = listdir + self._texts: dict[str, str] = {} + self._dirs: dict[str, dict[str, Entry]] = {} + + def read(self, path: str) -> str: + if path not in self._texts: + self._texts[path] = self._read(path) + return self._texts[path] + + def listdir(self, path: str) -> dict[str, Entry]: + if path not in self._dirs: + self._dirs[path] = self._listdir(path) + return self._dirs[path] + + def has(self, path: str) -> bool: + parent, _, name = path.rpartition("/") + try: + return name in self.listdir(parent) + except LauncherPatternMissing: + return False + + @classmethod + def from_files(cls, files: dict[str, str], links: dict[str, str] | None = None) -> RepoTree: + """A tree held in memory: file texts by path, symlink targets by path.""" + links = links or {} + + def read(path: str) -> str: + if path not in files: + raise LauncherPatternMissing(path, "file") + return files[path] + + def listdir(path: str) -> dict[str, Entry]: + prefix = path.rstrip("/") + "/" + entries: dict[str, Entry] = {} + for name, target in links.items(): + if name.startswith(prefix) and "/" not in name[len(prefix) :]: + entries[name[len(prefix) :]] = Entry("link", target) + for name in files: + if name.startswith(prefix): + rest = name[len(prefix) :] + head, _, tail = rest.partition("/") + entries.setdefault(head, Entry("dir" if tail else "file")) + if not entries: + raise LauncherPatternMissing(path, "directory") + return entries + + return cls(read, listdir) + + @classmethod + def github(cls, repo: str, rev: str) -> RepoTree: + """A GitHub repository at one revision: raw files and the trees API.""" + + def read(path: str) -> str: + url = f"https://raw.githubusercontent.com/{repo}/{rev}/{path}" + return fetch_bytes(url).decode("utf-8-sig") + + def listdir(path: str) -> dict[str, Entry]: + url = f"https://api.github.com/repos/{repo}/git/trees/{rev}:{path}" + try: + listing = json.loads(fetch_bytes(url, headers=github_headers())) + except ConnectionError as e: + raise LauncherPatternMissing(path, f"directory ({e})") from e + entries = {} + for item in listing.get("tree", []): + if item["mode"] == "120000": + entries[item["path"]] = Entry("link", read(f"{path}/{item['path']}").strip()) + else: + entries[item["path"]] = Entry("dir" if item["type"] == "tree" else "file") + return entries + + return cls(read, listdir) diff --git a/tests/test_launchers_rocknix.py b/tests/test_launchers_rocknix.py new file mode 100644 index 00000000..b49097ae --- /dev/null +++ b/tests/test_launchers_rocknix.py @@ -0,0 +1,593 @@ +"""ROCKNIX's launcher layer read from its packages. + +Each fixture is a cut-down copy of the lines an extractor reads at +ROCKNIX/distribution 20261001 (c445081a59518f37d9776e5412dd7b14910696f7). +The records must carry the names in the order the code writes or tries them, +point at the fixture's own lines, name a device only where the shipped +configs differ, and a construct that is gone must stop the scrape. +""" + +from __future__ import annotations + +import sys +import textwrap +import unittest +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from scraper import rocknix_launchers as rl # noqa: E402 +from scraper.rocknix_scraper import launcher_revision # noqa: E402 +from scraper.shell_launchers import LauncherPatternMissing, RepoTree # noqa: E402 + +SA = rl.STANDALONE + + +def text(block: str) -> str: + return textwrap.dedent(block).lstrip("\n") + + +def devices(*names: str) -> dict[str, str]: + return {f"{rl.DEVICES}/{name}/options": "" for name in names} + + +MELONDS_MK = text( + """ + makeinstall_target() { + mkdir -p ${INSTALL}/usr/config/melonDS + cp -a ${PKG_DIR}/config/${DEVICE}/* ${INSTALL}/usr/config/melonDS + } +""" +) + +MELONDS_INI = text( + """ + ExternalBIOSEnable=0 + BIOS9Path=/storage/roms/bios/bios9.bin + BIOS7Path=/storage/roms/bios/bios7.bin + FirmwarePath=/storage/roms/bios/firmware.bin + DSiBIOS9Path=/storage/roms/bios/dsi_bios9.bin + DSiBIOS7Path=/storage/roms/bios/dsi_bios7.bin + DSiFirmwarePath=/storage/roms/bios/dsi_firmware.bin + DSiNANDPath=/storage/roms/bios/dsi_nand.bin +""" +) + +MELONDS_AMD64_INI = text( + """ + ExternalBIOSEnable=0 + BIOS9Path= + BIOS7Path= + FirmwarePath= + DSiBIOS9Path= + DSiBIOS7Path= + DSiFirmwarePath= + DSiNANDPath= +""" +) + +MELONDS_SH = text( + """ + CONTYPE=$(get_setting console_type "${PLATFORM}" "${GAME}") + DBOOT=$(get_setting direct_boot "${PLATFORM}" "${GAME}") + if [ "$PLATFORM" = "ndsiware" ]; then + sed -i '/^DirectBoot=/c\\DirectBoot=0' /storage/.config/melonDS/melonDS.ini + else + if [ "$DBOOT" = "0" ]; then + sed -i '/^DirectBoot=/c\\DirectBoot=0' /storage/.config/melonDS/melonDS.ini + sed -i '/^ExternalBIOSEnable=/c\\ExternalBIOSEnable=1' /storage/.config/melonDS/melonDS.ini + else + sed -i '/^ExternalBIOSEnable=/c\\ExternalBIOSEnable=0' /storage/.config/melonDS/melonDS.ini + fi + fi +""" +) + + +def melonds_tree() -> RepoTree: + pkg = f"{SA}/melonds-sa" + return RepoTree.from_files( + { + **devices("AMD64", "H700", "RK3576", "S922X"), + f"{pkg}/package.mk": MELONDS_MK, + f"{pkg}/scripts/start_melonds.sh": MELONDS_SH, + f"{pkg}/config/AMD64/melonDS.ini": MELONDS_AMD64_INI, + f"{pkg}/config/InputPlumber/melonDS.ini": MELONDS_INI, + f"{pkg}/config/S922X/melonDS.ini": MELONDS_INI, + }, + links={f"{pkg}/config/RK3576": "InputPlumber"}, + ) + + +DSPERATE_MK = text( + """ + makeinstall_target() { + case ${DEVICE} in + RK3576|SM4450|SM6115|SM8250|SM8550|SM8650|SM8750) CONFIG="InputPlumber" ;; + *) CONFIG="${DEVICE}" ;; + esac + cp -a ${PKG_DIR}/config/${CONFIG}/* ${INSTALL}/usr/config/dsperate + } +""" +) + +DSPERATE_INI = text( + """ + [paths] + bios9 = /storage/roms/bios/bios9.bin + bios7 = /storage/roms/bios/bios7.bin + firmware = /storage/roms/bios/firmware.bin + bios9i = /storage/roms/bios/dsi_bios9.bin + bios7i = /storage/roms/bios/dsi_bios7.bin + dsi_firmware = /storage/roms/bios/dsi_firmware.bin + dsi_nand = /storage/roms/bios/dsi_nand.bin + games = /storage/roms/nds +""" +) + +AMIBERRY_SH = text( + """ + AMIBERRY_DIR="/storage/.config/amiberry" + AMIBERRY_CONFIG_DIR="${AMIBERRY_DIR}/conf" + BIOS_PATH="/storage/roms/bios" + AMIBERRY_SET_CONF() { + if [ `echo $1 | grep -E 'AGA|CD32' | wc -l` -eq 1 ]; then + cp ${AMIBERRY_CONFIG_DIR}/AmigaA1200-default.uae "${AMIBERRY_TMP_CONFIG}" + else + cp ${AMIBERRY_CONFIG_DIR}/AmigaA500-default.uae "${AMIBERRY_TMP_CONFIG}" + fi + sed -i "s#@BIOS_PATH@#${BIOS_PATH}#g" ${AMIBERRY_TMP_CONFIG} + } +""" +) + +AMIBERRY_MK = text( + """ + makeinstall_target() { + mkdir -p ${INSTALL}/usr/config/amiberry + cp -a ${PKG_DIR}/config/* ${INSTALL}/usr/config/amiberry + ln -s /storage/roms/bios ${INSTALL}/usr/config/amiberry/kickstarts + ln -s /usr/share/libretro/autoconfig ${INSTALL}/usr/config/amiberry/controller + } +""" +) + +MEDNAFEN_TEMPLATE = text( + """ + ;Path to firmware directory. + filesys.path_firmware /storage/roms/bios/ + + ;Path to PlayStation BIOS, Europe. + psx.bios_eu scph5502.bin + + ;Path to PlayStation BIOS, Japan. + psx.bios_jp scph5500.bin + + ;Enable BIOS sanity check. + psx.bios_sanity 1 + + ;Path to PC Engine CD BIOS. + pce.cdbios syscard3.pce +""" +) + +MEDNAFEN_SH = text( + """ + if [ ! -f "$MEDNAFEN_HOME/mednafen.cfg" ] + then + /usr/bin/bash /usr/bin/mednafen_gen_config.sh + fi +""" +) + +YABASANSHIRO_SH = text( + """ + BIOS="" + USE_BIOS=$(get_setting use_hlebios "${PLATFORM}" "${GAME}") + if [ "$USE_BIOS" = 1 ] + then + for BIOS in saturn_bios.bin sega_101.bin mpr-17933.bin mpr-18811-mx.ic1 mpr-19367-mx.ic1 stvbios.zip + do + BIOS=$(find /storage/roms/bios -name ${BIOS} -print 2>/dev/null) + if [ ! -z "${BIOS}" ] + then + BIOS="-b ${BIOS}" + break + fi + done + fi +""" +) + +DUCKSTATION_MK = text( + """ + makeinstall_target() { + cp -a ${PKG_DIR}/config/${DEVICE}/* ${INSTALL}/usr/config/duckstation + } +""" +) + +DRASTIC_SH = text( + """ + for bios in nds_bios_arm9.bin nds_bios_arm7.bin + do + if [ ! -e "/storage/.config/drastic/system/${bios}" ]; then + if [ -e "/storage/roms/bios/${bios}" ]; then + ln -sf /storage/roms/bios/${bios} /storage/.config/drastic/system + fi + fi + done +""" +) + +DOLPHIN_SH = text( + """ + CONF_DIR="/storage/.config/dolphin-emu" + ln -sf /storage/roms/savestates/gamecube ${CONF_DIR}/StateSaves + for REGION in EUR JAP USA + do + rm -rf "${CONF_DIR}/GC/${REGION}" + ln -sf "/storage/roms/bios/GC/${REGION}" "${CONF_DIR}/GC/${REGION}" + for SLOT in A B + do + MEM_CARD_FILE="MemoryCard${SLOT}.${REGION}.raw" + CONFIG_MEM_CARD="${CONF_DIR}/GC/${MEM_CARD_FILE}" + ROMS_BIOS_MEM_CARD="/storage/roms/bios/GC/${MEM_CARD_FILE}" + if [ -f "${ROMS_BIOS_MEM_CARD}" ]; then + ln -sf "${ROMS_BIOS_MEM_CARD}" "${CONFIG_MEM_CARD}" + elif [ -f "${CONFIG_MEM_CARD}" ]; then + ln -sf "${ROMS_BIOS_MEM_CARD}" "${CONFIG_MEM_CARD}" + fi + done + done +""" +) + +CEMU_SH = text( + """ + CEMU_CONFIG_ROOT="/storage/.config/Cemu" + CEMU_HOME_CONFIG="${CEMU_CONFIG_ROOT}/share" + CEMU_HOME_ONLINE="${CEMU_HOME_CONFIG}/online" + CEMU_HOME_KEYS="${CEMU_HOME_CONFIG}/keys" + CEMU_BIOS="/storage/roms/bios/cemu" + if [ ! -L ${CEMU_HOME_ONLINE} ]; then + ln -sf ${CEMU_BIOS}/online ${CEMU_HOME_ONLINE} + fi + if [ ! -L ${CEMU_HOME_KEYS} ]; then + ln -sf ${CEMU_BIOS}/keys ${CEMU_HOME_KEYS} + fi +""" +) + +RPCS3_SH = text( + """ + FOLDER_LINKS=("dev_flash" "dev_hdd0" "dev_hdd1" "custom_configs") + for FOLDER_LINK in "${FOLDER_LINKS[@]}"; do + TARGET_FOLDER="/storage/roms/bios/rpcs3/$FOLDER_LINK" + SOURCE_FOLDER="/storage/.config/rpcs3/$FOLDER_LINK" + rm -rf "$SOURCE_FOLDER" + ln -sf "$TARGET_FOLDER" "$SOURCE_FOLDER" + done +""" +) + +FLYCAST_SH = text( + """ + if [ ! -d "/storage/roms/bios/dc" ]; then + mkdir -p "/storage/roms/bios/dc" + fi + ln -sf "/storage/roms/bios/dc" "/storage/roms/dreamcast/data" +""" +) + + +def by(records: list[dict], emulator: str) -> list[dict]: + return [r for r in records if r["emulator"] == emulator] + + +class ConfigsPerDevice(unittest.TestCase): + def test_a_device_appears_only_where_configs_differ(self): + records = [r.entry() for r in rl.melonds(melonds_tree())] + self.assertEqual([r["device"] for r in records], ["AMD64", "RK3576, S922X"]) + empty, named = records + self.assertEqual(empty["names"], []) + self.assertTrue(empty["passes_to"].endswith("DSiNANDPath left empty")) + self.assertEqual( + named["names"], + [ + "bios9.bin", + "bios7.bin", + "firmware.bin", + "dsi_bios9.bin", + "dsi_bios7.bin", + "dsi_firmware.bin", + "dsi_nand.bin", + ], + ) + self.assertEqual(named["search_dir"], "/storage/roms/bios") + + def test_a_linked_device_directory_reads_its_target(self): + named = rl.melonds(melonds_tree())[1].entry() + self.assertIn(f"{SA}/melonds-sa/config/InputPlumber/melonDS.ini:2-8", named["source_ref"]) + self.assertNotIn("config/RK3576", named["source_ref"]) + + def test_the_external_bios_switch_is_the_option(self): + named = rl.melonds(melonds_tree())[1].entry() + self.assertEqual( + named["depends_on_option"], "direct_boot = 0 sets ExternalBIOSEnable=1, otherwise 0" + ) + self.assertIn(f"{SA}/melonds-sa/scripts/start_melonds.sh:2, 6-8", named["source_ref"]) + + def test_identical_configs_give_one_record_without_device(self): + pkg = f"{SA}/dsperate-sa" + tree = RepoTree.from_files( + { + **devices("H700", "RK3576", "S922X"), + f"{pkg}/package.mk": DSPERATE_MK, + f"{pkg}/scripts/start_dsperate.sh": '${EMUPERF} /usr/bin/dsperate --config "${DSPERATE_INI}" "${ROM}"\n', + f"{pkg}/config/H700/dsperate.ini": DSPERATE_INI, + f"{pkg}/config/InputPlumber/dsperate.ini": DSPERATE_INI, + f"{pkg}/config/S922X/dsperate.ini": DSPERATE_INI, + } + ) + (record,) = [r.entry() for r in rl.dsperate(tree)] + self.assertNotIn("device", record) + self.assertEqual( + record["passes_to"], + "dsperate.ini [paths] bios9, bios7, firmware, bios9i, bios7i, dsi_firmware, dsi_nand (--config)", + ) + # The case statement sends RK3576 to InputPlumber. + self.assertIn(f"{pkg}/config/InputPlumber/dsperate.ini:2-8", record["source_ref"]) + self.assertIn(f"{pkg}/package.mk:3-4", record["source_ref"]) + + def test_an_absent_key_and_an_empty_one_hand_over_the_same(self): + pkg = f"{SA}/duckstation-sa" + tree = RepoTree.from_files( + { + **devices("AMD64", "S922X"), + f"{pkg}/package.mk": DUCKSTATION_MK, + f"{pkg}/config/AMD64/settings.ini": "[BIOS]\nSearchDirectory = /storage/roms/bios\n", + f"{pkg}/config/S922X/settings.ini": text( + """ + [BIOS] + SearchDirectory = /storage/roms/bios + PathNTSCU = + PathNTSCJ = + PathPAL = + """ + ), + } + ) + (record,) = [r.entry() for r in rl.duckstation(tree)] + self.assertEqual(record["kind"], "scans_dir") + self.assertEqual(record["search_dir"], "/storage/roms/bios") + self.assertNotIn("device", record) + + def test_a_bios_written_into_a_path_key_is_an_unknown_shape(self): + pkg = f"{SA}/duckstation-sa" + tree = RepoTree.from_files( + { + **devices("S922X"), + f"{pkg}/package.mk": DUCKSTATION_MK, + f"{pkg}/config/S922X/settings.ini": "[BIOS]\nSearchDirectory = /storage/roms/bios\nPathNTSCU = scph5501.bin\n", + } + ) + with self.assertRaises(LauncherPatternMissing): + rl.duckstation(tree) + + def test_a_package_without_a_device_copy_raises(self): + pkg = f"{SA}/duckstation-sa" + tree = RepoTree.from_files( + { + **devices("S922X"), + f"{pkg}/package.mk": "makeinstall_target() {\n true\n}\n", + f"{pkg}/config/S922X/settings.ini": "[BIOS]\nSearchDirectory = /storage/roms/bios\n", + } + ) + with self.assertRaises(LauncherPatternMissing): + rl.duckstation(tree) + + +class NamesTheConfigsFix(unittest.TestCase): + def test_amiberry_machine_follows_the_game_name(self): + pkg = f"{SA}/amiberry" + uae = "amiberry.rom_path=/storage/.config/amiberry/kickstarts/\nkickstart_rom_file=@BIOS_PATH@/{}\nkickstart_ext_rom_file=\n" + tree = RepoTree.from_files( + { + f"{pkg}/scripts/start_amiberry.sh": AMIBERRY_SH, + f"{pkg}/package.mk": AMIBERRY_MK, + f"{pkg}/config/conf/AmigaA1200-default.uae": uae.format("kick40068.A1200.rom"), + f"{pkg}/config/conf/AmigaA500-default.uae": uae.format("kick34005.A500.rom"), + } + ) + a1200, a500, link = (r.entry() for r in rl.amiberry(tree)) + self.assertEqual((a1200["slot"], a1200["names"]), ("A1200", ["kick40068.A1200.rom"])) + self.assertEqual(a1200["depends_on_option"], "game name matches AGA|CD32") + self.assertEqual((a500["slot"], a500["names"]), ("A500", ["kick34005.A500.rom"])) + self.assertEqual(a500["depends_on_option"], "game name does not match AGA|CD32") + self.assertEqual(a500["search_dir"], "/storage/roms/bios") + self.assertEqual( + a1200["source_ref"], + f"{pkg}/scripts/start_amiberry.sh:3-10, {pkg}/config/conf/AmigaA1200-default.uae:2", + ) + self.assertEqual(link["kind"], "links") + self.assertEqual( + link["passes_to"], "symlink /usr/config/amiberry/kickstarts -> /storage/roms/bios" + ) + self.assertEqual(link["source_ref"], f"{pkg}/package.mk:4") + + def test_mednafen_one_record_per_module(self): + pkg = f"{SA}/mednafen" + tree = RepoTree.from_files( + { + f"{pkg}/config/common/mednafen.template": MEDNAFEN_TEMPLATE, + f"{pkg}/scripts/start_mednafen.sh": MEDNAFEN_SH, + } + ) + psx, pce = (r.entry() for r in rl.mednafen(tree)) + self.assertEqual((psx["slot"], psx["names"]), ("psx", ["scph5502.bin", "scph5500.bin"])) + self.assertEqual(psx["passes_to"], "mednafen.cfg psx.bios_eu, psx.bios_jp") + self.assertEqual(pce["names"], ["syscard3.pce"]) + self.assertEqual(psx["search_dir"], "/storage/roms/bios/") + self.assertEqual( + psx["source_ref"], + f"{pkg}/config/common/mednafen.template:2, 5, 8, {pkg}/scripts/start_mednafen.sh:1-3", + ) + + def test_mednafen_without_a_firmware_directory_raises(self): + pkg = f"{SA}/mednafen" + tree = RepoTree.from_files( + { + f"{pkg}/config/common/mednafen.template": "psx.bios_eu scph5502.bin\n", + f"{pkg}/scripts/start_mednafen.sh": MEDNAFEN_SH, + } + ) + with self.assertRaises(LauncherPatternMissing): + rl.mednafen(tree) + + +class SearchedNames(unittest.TestCase): + def tree(self, script: str) -> RepoTree: + return RepoTree.from_files({f"{SA}/yabasanshiro-sa/scripts/start_yabasanshiro.sh": script}) + + def test_yabasanshiro_tries_its_names_in_order(self): + (record,) = [r.entry() for r in rl.yabasanshiro(self.tree(YABASANSHIRO_SH))] + self.assertEqual(record["kind"], "names_searched") + self.assertEqual( + record["names"], + [ + "saturn_bios.bin", + "sega_101.bin", + "mpr-17933.bin", + "mpr-18811-mx.ic1", + "mpr-19367-mx.ic1", + "stvbios.zip", + ], + ) + self.assertEqual(record["passes_to"], "yabasanshiro -b ") + self.assertEqual(record["depends_on_option"], "use_hlebios = 1 (game, then platform)") + self.assertIs(record["case_insensitive"], False) + self.assertEqual( + record["source_ref"], f"{SA}/yabasanshiro-sa/scripts/start_yabasanshiro.sh:2-13" + ) + + def test_a_search_no_longer_written_as_a_loop_raises(self): + script = YABASANSHIRO_SH.replace("for BIOS in", "BIOS_LIST=").replace(" do\n", "") + with self.assertRaises(LauncherPatternMissing): + rl.yabasanshiro(self.tree(script)) + + +class Links(unittest.TestCase): + def all_scripts(self) -> dict[str, str]: + return { + "drastic": DRASTIC_SH, + "dolphin": DOLPHIN_SH, + "flycast": FLYCAST_SH, + "cemu": CEMU_SH, + "rpcs3": RPCS3_SH, + } + + def tree(self) -> RepoTree: + scripts = self.all_scripts() + return RepoTree.from_files({p: scripts[e] for e, p in rl.LINK_SCRIPTS}) + + def emulator(self, name: str) -> list[dict]: + return by([r.entry() for r in rl.links(self.tree())], name) + + def test_a_loop_names_each_file_and_links_into_a_directory(self): + (record,) = self.emulator("drastic") + self.assertEqual(record["names"], ["nds_bios_arm9.bin", "nds_bios_arm7.bin"]) + self.assertEqual(record["search_dir"], "/storage/roms/bios") + self.assertEqual( + record["passes_to"], + "symlink /storage/.config/drastic/system/ -> /storage/roms/bios/", + ) + # The link waits for an exact-case -e test. + self.assertIs(record["case_insensitive"], False) + self.assertEqual(record["source_ref"], f"{rl.LINK_SCRIPTS[0][1]}:1, 4-5") + + def test_a_nested_loop_is_a_placeholder_and_its_values(self): + regions, cards = self.emulator("dolphin") + self.assertEqual(regions["names"], ["EUR", "JAP", "USA"]) + self.assertEqual(regions["search_dir"], "/storage/roms/bios/GC") + self.assertEqual(cards["names"], ["MemoryCard{SLOT}.{REGION}.raw"]) + self.assertEqual(cards["expands"], {"REGION": ["EUR", "JAP", "USA"], "SLOT": ["A", "B"]}) + + def test_a_link_outside_the_bios_directory_is_not_a_record(self): + for record in self.emulator("dolphin"): + self.assertNotIn("StateSaves", record["passes_to"]) + + def test_one_by_one_links_of_one_layout_are_one_record(self): + (record,) = self.emulator("cemu") + self.assertEqual(record["names"], ["online", "keys"]) + self.assertEqual( + record["passes_to"], + "symlink /storage/.config/Cemu/share/ -> /storage/roms/bios/cemu/", + ) + + def test_an_array_loop_takes_the_array(self): + (record,) = self.emulator("rpcs3") + self.assertEqual(record["names"], ["dev_flash", "dev_hdd0", "dev_hdd1", "custom_configs"]) + self.assertEqual(record["search_dir"], "/storage/roms/bios/rpcs3") + self.assertEqual(record["source_ref"], f"{dict(rl.LINK_SCRIPTS)['rpcs3']}:1-6") + + def test_a_directory_link_has_no_names(self): + (record,) = self.emulator("flycast") + self.assertEqual(record["names"], []) + self.assertEqual( + record["passes_to"], "symlink /storage/roms/dreamcast/data -> /storage/roms/bios/dc" + ) + + def test_a_script_without_a_bios_link_raises(self): + scripts = self.all_scripts() + scripts["flycast"] = 'ln -sf "/storage/.config/flycast" "/storage/.local/share/flycast"\n' + tree = RepoTree.from_files({p: scripts[e] for e, p in rl.LINK_SCRIPTS}) + with self.assertRaises(LauncherPatternMissing): + rl.links(tree) + + +class Revision(unittest.TestCase): + def test_the_revision_is_the_one_the_bios_list_is_read_at(self): + url = ( + "https://raw.githubusercontent.com/ROCKNIX/distribution/20261001/" + "projects/ROCKNIX/packages/rocknix/sources/scripts/rocknix-systems" + ) + self.assertEqual(launcher_revision(url), "20261001") + + def test_a_url_without_a_revision_raises(self): + with self.assertRaises(ValueError): + launcher_revision("https://rocknix.org/") + + +class PlatformRecords(unittest.TestCase): + """The launchers platforms/rocknix.yml carries, as the scraper wrote them.""" + + @classmethod + def setUpClass(cls) -> None: + cls.config = yaml.safe_load((REPO_ROOT / "platforms" / "rocknix.yml").read_text()) + + def test_every_emulator_is_one_rocknix_packages(self): + cores = set(self.config["cores"]) + self.assertEqual(sorted({r["emulator"] for r in self.config["launchers"]} - cores), []) + + def test_every_reference_is_in_the_rocknix_project(self): + for record in self.config["launchers"]: + for part in record["source_ref"].split(", "): + if ":" in part: + self.assertTrue(part.startswith(f"{rl.ROCKNIX}/"), record) + + def test_a_device_is_named_only_beside_another_device_record(self): + with_device: dict[tuple[str, str], int] = {} + for record in self.config["launchers"]: + if "device" in record: + key = (record["emulator"], record["kind"]) + with_device[key] = with_device.get(key, 0) + 1 + self.assertTrue(with_device) + self.assertEqual({k: n for k, n in with_device.items() if n < 2}, {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index c842b494..6be79399 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -234,11 +234,29 @@ launchers: | `fails_when_missing` | `false`, or the message the launcher raises | | `fails_scope` | `emulator` when the failure needs every record of the emulator to come up empty | | `depends_on_option` | Options or conditions that decide whether the record applies | +| `device` | Devices whose shipped config the record reads, set only where a package ships configs that differ between devices | | `source_ref` | `path:lines` at the revision `source` pins | -Each record is read with `ast` from the generator. An extractor that no longer -recognises the shape of the code raises: an empty list would read as a -launcher that looks for nothing. +Batocera's generators are read with `ast`, RetroBat's C# generators as text. +ROCKNIX and EmuDeck launch from shell scripts and configs shipped with each +package, so `scripts/scraper/shell_launchers.py` reads assignments, `for` +loops, `ln -s` commands and `key = value` lines. A ROCKNIX package can ship +one config per device: the configs are grouped by the value read, and +`device` appears only when there is more than one group. melonDS writes its +seven BIOS paths on every device but AMD64, where the keys are left empty: + +```yaml +- emulator: melonds + build: standalone + kind: fixed_name + names: [] + passes_to: melonDS.ini BIOS9Path, BIOS7Path, FirmwarePath, DSiBIOS9Path, + DSiBIOS7Path, DSiFirmwarePath, DSiNANDPath left empty + device: AMD64 +``` + +An extractor that no longer recognises the shape of the code raises: an +empty list would read as a launcher that looks for nothing. ### Inheritance