feat: read ROCKNIX launchers from its packages

The scraper reads the start scripts and the configs each package installs at the tag the BIOS list is pinned to: configs that name a BIOS path, the yabasanshiro search, directories handed to DuckStation and the PCSX2 builds, and the ln -s commands that put bios/ where an emulator reads. A package that ships one config per device gets a device field only where the values differ, as melonDS does on AMD64.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 20:45:19 +02:00
1 parent a9c175d322
commit 57af8e4921
7 files changed
+1815 -4

No files matched your search

+4
View File
@@ -212,6 +212,10 @@
"type": "string",
"description": "Options or conditions in the launcher code that decide whether this record applies"
},
"device": {
"type": "string",
"description": "Devices whose shipped config this record reads, comma-separated, set only where the configs of a package differ between devices"
},
"source_ref": {
"type": "string",
"description": "path:lines in the platform's repository at the pinned revision"
+2 -1
View File
@@ -50,6 +50,7 @@ class Launcher:
fails_when_missing: bool | str = False
fails_scope: str | None = None
depends_on_option: str | None = None
device: str | None = None
def entry(self) -> dict[str, Any]:
"""The platform YAML form, in a fixed key order, unset keys left out."""
@@ -57,7 +58,7 @@ class Launcher:
"emulator", "build", "kind", "slot", "names", "expands",
"case_insensitive", "search_dir", "passes_to",
"fails_when_missing", "fails_scope", "depends_on_option",
"source_ref",
"device", "source_ref",
)
return {key: getattr(self, key) for key in order if getattr(self, key) is not None}
+648
View File
@@ -0,0 +1,648 @@
"""ROCKNIX's launcher layer, read from its packages at the pinned tag.
ROCKNIX starts each standalone emulator from a shell script and installs a
config with the package. Between a file in /storage/roms/bios and the
emulator, the config names the path (melonDS, xemu, mednafen), the script
searches or links (yabasanshiro, drastic, dolphin), or the emulator is
handed a directory and left to choose (DuckStation, AetherSX2). Several
packages ship one config per device: a record carries `device` only where
the value read differs between them.
"""
from __future__ import annotations
import posixpath
import re
from collections.abc import Callable
from dataclasses import dataclass
try:
from .batocera_launchers import Launcher
from .shell_launchers import (
LauncherPatternMissing,
Link,
RepoTree,
Script,
Setting,
describe_keys,
settings,
source_ref,
)
except ImportError:
from batocera_launchers import Launcher
from shell_launchers import (
LauncherPatternMissing,
Link,
RepoTree,
Script,
Setting,
describe_keys,
settings,
source_ref,
)
ROCKNIX = "projects/ROCKNIX"
DEVICES = f"{ROCKNIX}/devices"
STANDALONE = f"{ROCKNIX}/packages/emulators/standalone"
RETROARCH = f"{ROCKNIX}/packages/emulators/libretro/retroarch"
BIOS_ROOT = "/storage/roms/bios"
Lines = list[tuple[str, set[int]]]
@dataclass
class DeviceConfig:
"""The config file one or more devices install, and the lines that pick it."""
path: str
devices: list[str]
mk_lines: set[int]
def device_configs(
tree: RepoTree, package: str, file: str, folder: str = "config"
) -> list[DeviceConfig]:
"""Each distinct file package.mk installs as `file` for some device.
The directory comes from `${PKG_DIR}/<folder>/${DEVICE}/` or from a
`case ${DEVICE}` naming it. A device whose directory is absent does not
get the package. A directory that is a symlink resolves to its target.
"""
mk_path = f"{package}/package.mk"
mk = Script(mk_path, tree.read(mk_path))
arms: list[tuple[list[str], str]] = []
mk_lines: set[int] = set()
for block in re.finditer(r'case\s+"?\$\{?DEVICE\}?"?\s+in(.*?)\besac\b', mk.text, re.S):
body = block.group(1)
if not re.search(rf"CONFIG=|\$\{{PKG_DIR\}}/{folder}/", body):
continue
offset = block.start(1)
for arm in re.finditer(r"^\s*([\w*|]+)\)(.*?);;", body, re.M | re.S):
chosen = re.search(
rf'CONFIG="([^"]+)"|\$\{{PKG_DIR\}}/{folder}/([^/\s]+)', arm.group(2)
)
if chosen:
arms.append((arm.group(1).split("|"), chosen.group(1) or chosen.group(2)))
mk_lines.add(mk.text.count("\n", 0, offset + arm.start(2)) + 1)
break
if not arms:
number, _ = mk.find(
rf"\$\{{PKG_DIR\}}/{folder}/\$\{{DEVICE\}}/", f"{folder}/${{DEVICE}} copy"
)
arms, mk_lines = [(["*"], "${DEVICE}")], {number}
root = f"{package}/{folder}"
entries = tree.listdir(root)
by_path: dict[str, DeviceConfig] = {}
for device, entry in tree.listdir(DEVICES).items():
if entry.kind != "dir":
continue
arm = next((d for patterns, d in arms if device in patterns or "*" in patterns), None)
if arm is None:
continue
name = device if arm in ("${DEVICE}", "${CONFIG}") else arm
found = entries.get(name)
if found is None:
continue
if found.kind == "link":
name = found.target.rstrip("/")
path = f"{root}/{name}/{file}"
by_path.setdefault(path, DeviceConfig(path, [], mk_lines)).devices.append(device)
if not by_path:
raise LauncherPatternMissing(root, f"{file} for any device")
return sorted(by_path.values(), key=lambda c: c.path)
def per_device(
tree: RepoTree,
configs: list[DeviceConfig],
mk_path: str,
read: Callable[[str, str], tuple[object, set[int]]],
) -> list[tuple[str | None, object, Lines]]:
"""Configs grouped by what read() finds in them.
read(path, text) returns a comparable value and the lines it comes from.
One group gives one result without a device. Otherwise each group names
its devices, in the order of the devices directory.
"""
groups: dict[object, tuple[list[str], Lines]] = {}
for config in configs:
value, lines = read(config.path, tree.read(config.path))
devices, refs = groups.setdefault(value, ([], []))
devices.extend(config.devices)
refs.append((config.path, lines))
mk_lines = set().union(*(c.mk_lines for c in configs))
if len(groups) == 1:
((value, (_, refs)),) = groups.items()
return [(None, value, [*refs, (mk_path, mk_lines)])]
order = list(tree.listdir(DEVICES))
return [
(", ".join(sorted(devices, key=order.index)), value, [*refs, (mk_path, mk_lines)])
for value, (devices, refs) in sorted(
groups.items(), key=lambda g: min(order.index(d) for d in g[1][0])
)
]
def _file_under(value: str, root: str) -> bool:
return value.startswith(root.rstrip("/") + "/")
@dataclass(frozen=True)
class FixedPaths:
"""Keys of a config that name a file under one directory."""
keys: tuple[tuple[str, str], ...]
names: tuple[str, ...]
search_dir: str
def fixed_paths(
found: list[Setting], keys: list[tuple[str, str]], path: str
) -> tuple[FixedPaths, set[int]]:
"""The values of keys, as names relative to the directory they share."""
by_key = {(s.section, s.key): s for s in found}
missing = [k for k in keys if k not in by_key]
if missing:
raise LauncherPatternMissing(path, f"key {missing[0][1]}")
values = [by_key[k].value for k in keys if by_key[k].value]
lines = {by_key[k].line for k in keys}
if not values:
return FixedPaths(tuple(keys), (), ""), lines
folder = posixpath.commonpath([posixpath.dirname(v) for v in values])
names = tuple(posixpath.relpath(v, folder) for v in values)
return FixedPaths(tuple(keys), names, folder), lines
def bios_keys(tree: RepoTree, paths: list[str], root: str = BIOS_ROOT) -> list[tuple[str, str]]:
"""Keys whose value is a file under root in at least one of the configs."""
keys: list[tuple[str, str]] = []
for path in paths:
for setting in settings(tree.read(path)):
key = (setting.section, setting.key)
if _file_under(setting.value, root) and key not in keys:
keys.append(key)
if not keys:
raise LauncherPatternMissing(paths[0], f"key naming a file under {root}")
return keys
def config_records(
tree: RepoTree,
emulator: str,
package: str,
file: str,
passes_to_suffix: str = "",
**fields: object,
) -> list[Launcher]:
"""fixed_name records for the bios paths of a config installed per device."""
configs = device_configs(tree, package, file)
keys = bios_keys(tree, [c.path for c in configs])
name = posixpath.basename(file)
def read(path: str, text: str) -> tuple[object, set[int]]:
return fixed_paths(settings(text), keys, path)
records = []
for device, value, refs in per_device(tree, configs, f"{package}/package.mk", read):
assert isinstance(value, FixedPaths)
described = describe_keys(name, value.keys)
records.append(
Launcher(
emulator=emulator,
build="standalone",
kind="fixed_name",
names=list(value.names),
source_ref=source_ref(refs),
search_dir=value.search_dir or None,
passes_to=(
(described + passes_to_suffix) if value.names else f"{described} left empty"
),
device=device,
**fields,
)
)
return records
def retroarch(tree: RepoTree) -> list[Launcher]:
"""system_directory of the retroarch.cfg each device installs."""
configs = device_configs(tree, RETROARCH, "retroarch.cfg", folder="sources")
wanted = ("system_directory", "systemfiles_in_content_dir")
def read(path: str, text: str) -> tuple[object, set[int]]:
found = {s.key: s for s in settings(text) if s.key in wanted}
if "system_directory" not in found:
raise LauncherPatternMissing(path, "system_directory")
return tuple((k, found[k].value) for k in wanted if k in found), {
s.line for s in found.values()
}
records = []
for device, value, refs in per_device(tree, configs, f"{RETROARCH}/package.mk", read):
assert isinstance(value, tuple)
values = dict(value)
records.append(
Launcher(
emulator="retroarch",
build="libretro",
kind="passes_path",
names=[],
source_ref=source_ref(refs),
search_dir=values["system_directory"],
passes_to="retroarch.cfg " + ", ".join(f'{k} = "{v}"' for k, v in value),
device=device,
)
)
return records
def amiberry(tree: RepoTree) -> list[Launcher]:
"""The A500 or A1200 config, chosen by the game name, then the kickstarts link."""
package = f"{STANDALONE}/amiberry"
script = Script(
f"{package}/scripts/start_amiberry.sh", tree.read(f"{package}/scripts/start_amiberry.sh")
)
test_line, test = script.find(r"grep -E '([^']+)'", "machine choice by game name")
then_line, then = script.find(
r"cp \S+/(\w+-default\.uae)", "config copied for a match", test_line
)
else_line, _ = script.find(r"^\s*else\b", "else of the machine choice", then_line)
other_line, other = script.find(
r"cp \S+/(\w+-default\.uae)", "config copied otherwise", else_line
)
sed_line, sed = script.find(
r'sed -i "s#(@\w+@)#\$\{(\w+)\}#g"', "placeholder substitution", other_line
)
used: set[int] = {test_line, then_line, else_line, other_line, sed_line}
root = script.expand("${" + sed.group(2) + "}", sed_line, used)
records = []
for config, condition in (
(then.group(1), f"game name matches {test.group(1)}"),
(other.group(1), f"game name does not match {test.group(1)}"),
):
path = f"{package}/config/conf/{config}"
found = [
Setting(s.section, s.key, s.value.replace(sed.group(1), root), s.line)
for s in settings(tree.read(path))
]
keys = [(s.section, s.key) for s in found if _file_under(s.value, root)]
if not keys:
raise LauncherPatternMissing(path, f"key naming a file under {sed.group(1)}")
value, lines = fixed_paths(found, keys, path)
slot = re.fullmatch(r"Amiga(\w+)-default\.uae", config)
records.append(
Launcher(
emulator="amiberry",
build="standalone",
kind="fixed_name",
names=list(value.names),
source_ref=source_ref([(script.path, used), (path, lines)]),
slot=slot.group(1) if slot else config,
search_dir=value.search_dir,
passes_to=f"{describe_keys(config, value.keys)} (amiberry -f)",
depends_on_option=condition,
)
)
mk = Script(f"{package}/package.mk", tree.read(f"{package}/package.mk"))
links = mk.links(BIOS_ROOT, fixed={"INSTALL": ""})
if not links:
raise mk.missing(f"ln -s of {BIOS_ROOT}")
records.extend(link_record("amiberry", mk, link) for link in links)
return records
def link_record(emulator: str, script: Script, link: Link, **fields: object) -> Launcher:
return Launcher(
emulator=emulator,
build="standalone",
kind="links",
names=list(link.names),
source_ref=source_ref([(script.path, link.lines)]),
expands=link.expands or None,
search_dir=link.bios_side,
passes_to=f"symlink {link.link} -> {link.target}",
**fields,
)
def hatari(tree: RepoTree) -> list[Launcher]:
package = f"{STANDALONE}/hatarisa"
script = Script(
f"{package}/scripts/start_hatari.sh", tree.read(f"{package}/scripts/start_hatari.sh")
)
copy_line, copy = script.find(r"cp \S+/([\w-]+\.cfg) ", "default config copy")
run_line, _ = script.find(r"--configfile", "--configfile", copy_line)
path = f"{package}/config/conf/{copy.group(1)}"
found = settings(tree.read(path))
keys = [(s.section, s.key) for s in found if _file_under(s.value, BIOS_ROOT)]
if not keys:
raise LauncherPatternMissing(path, f"key naming a file under {BIOS_ROOT}")
value, lines = fixed_paths(found, keys, path)
return [
Launcher(
emulator="hatari",
build="standalone",
kind="fixed_name",
names=list(value.names),
source_ref=source_ref([(path, lines), (script.path, {copy_line, run_line})]),
search_dir=value.search_dir,
passes_to=f"{describe_keys(copy.group(1), value.keys)} (--configfile)",
)
]
def melonds(tree: RepoTree) -> list[Launcher]:
package = f"{STANDALONE}/melonds-sa"
script = Script(
f"{package}/scripts/start_melonds.sh", tree.read(f"{package}/scripts/start_melonds.sh")
)
enable_line, _ = script.find(r"ExternalBIOSEnable=1", "ExternalBIOSEnable=1")
tests = [
(number, found)
for number, line in script.code()
if number < enable_line and (found := re.search(r'if \[ "\$(\w+)" = "(\w+)" \]', line))
]
if not tests:
raise script.missing("test that enables the external BIOS")
test_line, test = tests[-1]
setting_line, setting = script.find(
rf"^\s*{test.group(1)}=\$\(get_setting (\w+) ", f"setting read into {test.group(1)}"
)
condition = f"{setting.group(1)} = {test.group(2)} sets ExternalBIOSEnable=1, otherwise 0"
records = config_records(tree, "melonds", package, "melonDS.ini", depends_on_option=condition)
for record in records:
record.source_ref += (
f", {source_ref([(script.path, {setting_line, test_line, enable_line})])}"
)
return records
def dsperate(tree: RepoTree) -> list[Launcher]:
package = f"{STANDALONE}/dsperate-sa"
script = Script(
f"{package}/scripts/start_dsperate.sh", tree.read(f"{package}/scripts/start_dsperate.sh")
)
run_line, _ = script.find(r"/usr/bin/dsperate --config ", "dsperate --config")
records = config_records(tree, "dsperate", package, "dsperate.ini", " (--config)")
for record in records:
record.source_ref += f", {source_ref([(script.path, {run_line})])}"
return records
def xemu(tree: RepoTree) -> list[Launcher]:
package = f"{STANDALONE}/xemu-sa"
script = Script(
f"{package}/scripts/start_xemu.sh", tree.read(f"{package}/scripts/start_xemu.sh")
)
copy_line, _ = script.find(
r'cp -r "\$\{SOURCE_DIR\}/\$\{XEMU_INI\}"', "xemu.toml copy at launch"
)
records = config_records(tree, "xemu", package, "xemu.toml", " (copied at each launch)")
for record in records:
record.source_ref += f", {source_ref([(script.path, {copy_line})])}"
return records
def nanoboyadvance(tree: RepoTree) -> list[Launcher]:
package = f"{STANDALONE}/nanoboyadvance-sa"
path = f"{package}/config/config.toml"
found = settings(tree.read(path))
keys = [(s.section, s.key) for s in found if _file_under(s.value, BIOS_ROOT)]
if not keys:
raise LauncherPatternMissing(path, f"key naming a file under {BIOS_ROOT}")
value, lines = fixed_paths(found, keys, path)
mk_path = f"{package}/package.mk"
mk_line, _ = Script(mk_path, tree.read(mk_path)).find(
r"config/config\.toml", "config.toml install"
)
return [
Launcher(
emulator="nanoboyadvance",
build="standalone",
kind="fixed_name",
names=list(value.names),
source_ref=source_ref([(path, lines), (mk_path, {mk_line})]),
search_dir=value.search_dir,
passes_to=describe_keys("config.toml", value.keys),
)
]
def mednafen(tree: RepoTree) -> list[Launcher]:
"""The firmware keys of mednafen.template, one record per module."""
package = f"{STANDALONE}/mednafen"
path = f"{package}/config/common/mednafen.template"
found = settings(tree.read(path), separator=" ")
folder = next((s for s in found if s.key == "filesys.path_firmware"), None)
if folder is None:
raise LauncherPatternMissing(path, "filesys.path_firmware")
pattern = re.compile(r"(\w+)\.(\w*bios\w*|ggrom|cart\.\w+_path)")
firmware = [s for s in found if pattern.fullmatch(s.key) and s.value and not s.value.isdigit()]
if not firmware:
raise LauncherPatternMissing(path, "firmware keys")
script_path = f"{package}/scripts/start_mednafen.sh"
script = Script(script_path, tree.read(script_path))
gen_line, _ = script.find(r"mednafen_gen_config\.sh", "config generation")
test_line, _ = script.find(
r'if \[ ! -f "\$MEDNAFEN_HOME/mednafen\.cfg" \]', "mednafen.cfg test"
)
modules: dict[str, list[Setting]] = {}
for setting in firmware:
modules.setdefault(setting.key.split(".", 1)[0], []).append(setting)
return [
Launcher(
emulator="mednafen",
build="standalone",
kind="fixed_name",
names=list(dict.fromkeys(s.value for s in group)),
source_ref=source_ref(
[
(path, {folder.line, *(s.line for s in group)}),
(script_path, {test_line, gen_line}),
]
),
slot=module,
search_dir=folder.value,
passes_to="mednafen.cfg " + ", ".join(s.key for s in group),
)
for module, group in modules.items()
]
def yabasanshiro(tree: RepoTree) -> list[Launcher]:
package = f"{STANDALONE}/yabasanshiro-sa"
script = Script(
f"{package}/scripts/start_yabasanshiro.sh",
tree.read(f"{package}/scripts/start_yabasanshiro.sh"),
)
option_line, option = script.find(r"=\$\(get_setting (\w+) ", "BIOS option")
test_line, test = script.find(r'if \[ "\$\w+" = (\w+) \]', "BIOS option test", option_line)
loop = next((lp for lp in script.loops() if lp.start > test_line), None)
if loop is None:
raise script.missing("loop over BIOS names")
find_line, find = script.find(
rf"\$\(find (\S+) -name \$\{{{loop.var}\}}", "find of each name", loop.start, loop.end
)
flag_line, flag = script.find(
r'="(-\w) \$\{\w+\}"', "flag the found path is passed with", find_line, loop.end
)
break_line, _ = script.find(r"^\s*break\b", "break on the first found", flag_line, loop.end)
return [
Launcher(
emulator="yabasanshiro",
build="standalone",
kind="names_searched",
names=loop.items,
source_ref=source_ref(
[(script.path, {option_line, test_line, *range(loop.start, loop.end + 1)})]
),
case_insensitive=False,
search_dir=f"{find.group(1)} (recursive)",
passes_to=f"yabasanshiro {flag.group(1)} <first path found>",
fails_when_missing=False,
depends_on_option=f"{option.group(1)} = {test.group(1)} (game, then platform)",
)
]
def directory_records(
tree: RepoTree,
emulator: str,
package: str,
file: str,
folder_key: tuple[str, str],
file_keys: tuple[tuple[str, str], ...],
) -> list[Launcher]:
"""scans_dir records: the config hands a directory and leaves the file keys empty."""
configs = device_configs(tree, package, file)
name = posixpath.basename(file)
def read(path: str, text: str) -> tuple[object, set[int]]:
found = {(s.section, s.key): s for s in settings(text)}
if folder_key not in found:
raise LauncherPatternMissing(path, f"[{folder_key[0]}] {folder_key[1]}")
set_files = [k for k in file_keys if k in found and found[k].value]
if set_files:
raise LauncherPatternMissing(path, f"empty [{set_files[0][0]}] {set_files[0][1]}")
lines = {found[k].line for k in (folder_key, *file_keys) if k in found}
return found[folder_key].value, lines
unset = describe_keys("", file_keys).strip()
records = []
for device, folder, refs in per_device(tree, configs, f"{package}/package.mk", read):
passes = f"{name} [{folder_key[0]}] {folder_key[1]} = {folder}, {unset} unset"
records.append(
Launcher(
emulator=emulator,
build="standalone",
kind="scans_dir",
names=[],
source_ref=source_ref(refs),
search_dir=folder,
passes_to=passes,
device=device,
)
)
return records
def duckstation(tree: RepoTree) -> list[Launcher]:
return directory_records(
tree,
"duckstation",
f"{STANDALONE}/duckstation-sa",
"settings.ini",
("BIOS", "SearchDirectory"),
(("BIOS", "PathNTSCU"), ("BIOS", "PathNTSCJ"), ("BIOS", "PathPAL")),
)
def aethersx2(tree: RepoTree) -> list[Launcher]:
return directory_records(
tree,
"aethersx2",
f"{STANDALONE}/aethersx2-sa",
"aethersx2/inis/PCSX2.ini",
("Folders", "Bios"),
(("Filenames", "BIOS"),),
)
def armsx2(tree: RepoTree) -> list[Launcher]:
return directory_records(
tree,
"armsx2",
f"{STANDALONE}/armsx2-sa",
"ARMSX2/inis/PCSX2.ini",
("Folders", "Bios"),
(("Filenames", "BIOS"),),
)
# Start scripts whose ln -s commands put a BIOS path where the emulator reads.
LINK_SCRIPTS = (
("drastic", f"{STANDALONE}/drastic-sa/scripts/start_drastic.sh"),
("dolphin", f"{STANDALONE}/dolphin-sa/scripts/start_dolphin_gc.sh"),
("flycast", f"{STANDALONE}/flycast-sa/scripts/start_flycast.sh"),
("cemu", f"{STANDALONE}/cemu-sa/scripts/start_cemu.sh"),
("rpcs3", f"{STANDALONE}/rpcs3-sa/scripts/start_rpcs3.sh"),
)
def links(tree: RepoTree) -> list[Launcher]:
records = []
for emulator, path in LINK_SCRIPTS:
script = Script(path, tree.read(path))
found = script.links(BIOS_ROOT)
if not found:
raise script.missing(f"ln -s of a path under {BIOS_ROOT}")
for link in found:
guard = _existence_guard(script, link)
if guard is not None:
link.lines.add(guard)
records.append(
link_record(
emulator,
script,
link,
case_insensitive=False if guard is not None else None,
fails_when_missing=False,
)
)
return records
def _existence_guard(script: Script, link: Link) -> int | None:
"""The line of a -e test of the BIOS-side path the link waits for, if any."""
last = max(link.lines)
tests = [
number
for number, line in script.code()
if number < last
and re.search(
r"\[ -e \S*" + re.escape(link.bios_side) + "/", script.expand(line, number, set())
)
]
return tests[-1] if tests and last - tests[-1] <= 3 else None
EXTRACTORS: tuple[Callable[[RepoTree], list[Launcher]], ...] = (
retroarch,
amiberry,
hatari,
melonds,
dsperate,
nanoboyadvance,
xemu,
mednafen,
yabasanshiro,
duckstation,
aethersx2,
armsx2,
links,
)
def extract_launchers(tree: RepoTree) -> list[dict]:
"""Every launcher record the extractors find, in extractor order."""
return [record.entry() for extractor in EXTRACTORS for record in extractor(tree)]
+16
View File
@@ -24,6 +24,8 @@ try:
fetch_github_latest_version,
requirement_entry,
)
from .rocknix_launchers import extract_launchers
from .shell_launchers import RepoTree
except ImportError:
from base_scraper import (
BaseScraper,
@@ -31,6 +33,8 @@ except ImportError:
fetch_github_latest_version,
requirement_entry,
)
from rocknix_launchers import extract_launchers
from shell_launchers import RepoTree
PLATFORM_NAME = "rocknix"
@@ -53,6 +57,14 @@ STANDALONE_TREE_URL = (
)
def launcher_revision(url: str) -> str:
"""The tag or commit a rocknix-systems URL is read at."""
found = re.search(r"/ROCKNIX/distribution/([^/]+)/projects/", url)
if found is None:
raise ValueError(f"{url} names no ROCKNIX/distribution revision")
return found.group(1)
def standalone_emulators(package_names: list[str]) -> list[str]:
"""The standalone emulators ROCKNIX packages, by the names profiles use.
@@ -274,6 +286,10 @@ class Scraper(BaseScraper):
# emulator as a RetroArch core.
"dual_mode_cores": dual_mode_emulators(standalone, self._fetch_emulators_mk()),
"systems": systems,
# Read at the revision the BIOS list comes from.
"launchers": extract_launchers(
RepoTree.github(GITHUB_REPO, launcher_revision(self.url))
),
}
+531
View File
@@ -0,0 +1,531 @@
"""Readers for launchers written as shell scripts and key=value configs.
ROCKNIX and EmuDeck launch emulators from bash: a start script links or
writes a BIOS path, a config shipped with the package names the file. These
helpers read that text without running it. Each one targets a construct
(an assignment, a for loop, an ln command, a key of a config) and the
extractors built on them raise LauncherPatternMissing when the construct is
gone, so a rewrite upstream stops the scrape instead of publishing nothing.
"""
from __future__ import annotations
import json
import re
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
try:
from .base_scraper import fetch_bytes, github_headers
from .batocera_launchers import LauncherShapeError
except ImportError:
from base_scraper import fetch_bytes, github_headers
from batocera_launchers import LauncherShapeError
NAME = r"[A-Za-z_][A-Za-z0-9_]*"
# A single-quoted "$" is literal: it is carried as this mark through expansion.
LITERAL_DOLLAR = "\x00"
# A loop variable expands to this mark around its name, so the caller can
# tell which part of a path the loop supplies.
LOOP_MARK = "\x01"
class LauncherPatternMissing(LauncherShapeError):
"""A construct an extractor reads is absent from the launcher source."""
def __init__(self, path: str, what: str) -> None:
super().__init__(f"{path}: {what} not found")
def span(first: int, last: int) -> str:
return str(first) if first == last else f"{first}-{last}"
def merge_lines(lines: Iterable[int]) -> list[tuple[int, int]]:
"""Line numbers as sorted spans, a gap of one line joined."""
spans: list[tuple[int, int]] = []
for line in sorted(set(lines)):
if spans and line <= spans[-1][1] + 2:
spans[-1] = (spans[-1][0], line)
else:
spans.append((line, line))
return spans
def source_ref(parts: Iterable[tuple[str, Iterable[int]]]) -> str:
"""path:a-b, c for each file, files in the order given."""
merged: dict[str, set[int]] = {}
for path, lines in parts:
merged.setdefault(path, set()).update(lines)
return ", ".join(
f"{path}:{', '.join(span(*s) for s in merge_lines(lines))}"
for path, lines in merged.items()
)
@dataclass
class Word:
"""A shell word: as written, and with quotes removed."""
raw: str
text: str
def words(line: str) -> list[Word]:
"""The words of one command line, quotes removed, comments dropped.
Variables are left unexpanded, each written ${NAME} so the quoting that
ended a name ("$a""b") still ends it. A "$" inside single quotes is kept
as LITERAL_DOLLAR so a later expansion leaves it alone.
"""
result: list[Word] = []
raw, text, quote, started, segment = "", "", "", False, 0
def close_segment() -> None:
nonlocal text, segment
text = text[:segment] + re.sub(rf"\$({NAME})", r"${\1}", text[segment:])
segment = len(text)
i = 0
while i < len(line):
char = line[i]
if quote == "'":
raw += char
if char == "'":
quote = ""
segment = len(text)
else:
text += LITERAL_DOLLAR if char == "$" else char
elif quote == '"':
raw += char
if char == "\\" and i + 1 < len(line) and line[i + 1] in '"\\$`':
raw += line[i + 1]
text += LITERAL_DOLLAR if line[i + 1] == "$" else line[i + 1]
i += 1
elif char == '"':
quote = ""
close_segment()
else:
text += char
elif char in "'\"":
close_segment()
quote, started = char, True
raw += char
elif char == "\\" and i + 1 < len(line):
raw += char + line[i + 1]
text += line[i + 1]
started = True
i += 1
elif char == "#" and not started:
break
elif char.isspace() or char == ";":
if started:
close_segment()
result.append(Word(raw, text))
raw, text, started, segment = "", "", False, 0
if char == ";":
result.append(Word(";", ";"))
else:
raw += char
text += char
started = True
i += 1
if started:
close_segment()
result.append(Word(raw, text))
return result
@dataclass
class Assignment:
line: int
value: str
@dataclass
class Loop:
"""A for loop: its variable, its items and the lines they come from."""
var: str
items: list[str]
start: int
end: int
lines: set[int] = field(default_factory=set)
@dataclass
class Link:
"""An ln -s whose link or target lies under the BIOS root."""
target: str
link: str
bios_side: str
names: list[str]
lines: set[int]
bios_is_target: bool = True
expands: dict[str, list[str]] = field(default_factory=dict)
def _placeholders(text: str, marks: dict[str, Loop]) -> str:
"""Loop marks written as {VAR}, the placeholder `expands` resolves."""
for mark, loop in marks.items():
text = text.replace(mark, "{" + loop.var + "}")
return text
class Script:
"""One shell script or makefile, read line by line."""
def __init__(self, path: str, text: str) -> None:
self.path = path
self.text = text
self.lines = text.splitlines()
self._assignments: dict[str, list[Assignment]] = {}
self._arrays: dict[str, tuple[int, list[str]]] = {}
for number, line in enumerate(self.lines, 1):
found = re.match(rf"\s*(?:local\s+|export\s+)?({NAME})=(.*)$", line)
if found is None:
continue
array = re.fullmatch(r"\((.*)\)\s*", found.group(2))
if array:
self._arrays[found.group(1)] = (number, [w.text for w in words(array.group(1))])
continue
value = words(found.group(2))
if value and value[0].raw:
self._assignments.setdefault(found.group(1), []).append(
Assignment(number, value[0].text)
)
def missing(self, what: str) -> LauncherPatternMissing:
return LauncherPatternMissing(self.path, what)
def code(self) -> Iterable[tuple[int, str]]:
"""Lines that are not comments."""
for number, line in enumerate(self.lines, 1):
if not line.lstrip().startswith("#"):
yield number, line
def find(
self, pattern: str, what: str, start: int = 1, end: int | None = None
) -> tuple[int, re.Match[str]]:
"""The first code line from start to end matching pattern."""
for number, line in self.code():
if number < start or (end is not None and number > end):
continue
found = re.search(pattern, line)
if found:
return number, found
raise self.missing(what)
def assignment_ending(self, suffix: str) -> Assignment | None:
"""The first assignment whose value ends with suffix."""
return next(
(a for found in self._assignments.values() for a in found if a.value.endswith(suffix)),
None,
)
def value(self, name: str, before: int) -> Assignment | None:
"""The last assignment of name above a line, else the first below it."""
assignments = self._assignments.get(name, [])
above = [a for a in assignments if a.line < before]
if above:
return above[-1]
return assignments[0] if assignments else None
def expand(
self, text: str, line: int, used: set[int], fixed: dict[str, str] | None = None
) -> str:
"""Variables replaced by their assigned value, unknown ones as ${NAME}."""
fixed = fixed or {}
def replace(found: re.Match[str], depth: int) -> str:
name = found.group(1) or found.group(2)
if name in fixed:
return fixed[name]
assignment = self.value(name, line)
if assignment is None or depth > 8:
return "${" + name + "}"
used.add(assignment.line)
return expand_once(assignment.value, assignment.line, depth + 1)
def expand_once(value: str, at: int, depth: int) -> str:
nonlocal line
saved, line = line, at
try:
return re.sub(rf"\$\{{({NAME})\}}|\$({NAME})", lambda m: replace(m, depth), value)
finally:
line = saved
return expand_once(text, line, 0).replace(LITERAL_DOLLAR, "$")
def loops(self) -> list[Loop]:
"""Every for loop, items from a word list or from "${ARRAY[@]}"."""
found_loops = []
for number, line in self.code():
head = re.match(rf"\s*for ({NAME}) in (.*?)\s*(?:;\s*do)?\s*$", line)
if head is None:
continue
items = [w.text for w in words(head.group(2)) if w.text != ";"]
lines = {number}
if len(items) == 1 and (array := re.fullmatch(rf"\$\{{({NAME})\[@\]\}}", items[0])):
if array.group(1) not in self._arrays:
raise self.missing(f"array {array.group(1)}")
array_line, items = self._arrays[array.group(1)]
lines.add(array_line)
found_loops.append(Loop(head.group(1), items, number, self._loop_end(number), lines))
return found_loops
def _loop_end(self, start: int) -> int:
depth = 0
for number, line in self.code():
if number < start:
continue
if re.match(r"\s*(for|while|until)\b", line):
depth += 1
if re.match(r"\s*done\b", line) or re.search(r";\s*done\s*$", line):
depth -= 1
if depth == 0:
return number
raise self.missing(f"done of the loop at line {start}")
def links(self, bios_root: str, fixed: dict[str, str] | None = None) -> list[Link]:
"""ln -s commands with the link or its target under bios_root.
Links in one loop, or written one by one as P/n linked to Q/n, are
one Link listing each n.
"""
loops = self.loops()
found: list[Link] = []
for number, line in self.code():
command = words(line)
heads = [
i
for i, w in enumerate(command)
if w.raw == "ln"
and (i == 0 or command[i - 1].raw in {";", "then", "do", "&&", "||"})
]
for head in heads:
args = []
for word in command[head + 1 :]:
if word.raw in {";", "&&", "||"}:
break
args.append(word)
options = [w.text for w in args if w.text.startswith("-")]
operands = [w.text for w in args if not w.text.startswith("-")]
if not any("s" in o for o in options) or len(operands) != 2:
continue
enclosing = [lp for lp in loops if lp.start < number <= lp.end]
env = dict(fixed or {})
marks = {f"{LOOP_MARK}{lp.var}{LOOP_MARK}": lp for lp in enclosing}
env.update({lp.var: mark for mark, lp in marks.items()})
used = {number}
target, link = (self.expand(op, number, used, env) for op in operands)
if not (_under(target, bios_root) or _under(link, bios_root)):
continue
bios_is_target = _under(target, bios_root)
bios_side = (target if bios_is_target else link).rstrip("/")
folder, _, last = bios_side.rpartition("/")
present = [m for m in marks if m in target or m in link]
if any(m in folder for m in present):
raise self.missing(
f"loop variable only in the last segment of the ln at line {number}"
)
names: list[str] = []
expands: dict[str, list[str]] = {}
if present:
bios_side = folder
if last in marks:
names = list(marks[last].items)
else:
names = [_placeholders(last, marks)]
expands = {lp.var: list(lp.items) for m, lp in marks.items() if m in last}
if bios_is_target and last not in link:
link = f"{link.rstrip('/')}/{last}"
for mark in present:
used |= marks[mark].lines
template = _placeholders(last, marks) if expands else "<name>"
target, link = (
p.replace(last, template) if last in p else _placeholders(p, marks)
for p in (target, link)
)
duplicate = next((f for f in found if (f.target, f.link) == (target, link)), None)
if duplicate is not None:
duplicate.lines |= used
continue
found.append(Link(target, link, bios_side, names, used, bios_is_target, expands))
return _merge_links(found)
def _under(path: str, root: str) -> bool:
return path.rstrip("/") == root.rstrip("/") or path.startswith(root.rstrip("/") + "/")
def _merge_links(links: list[Link]) -> list[Link]:
"""Single links P/n -> Q/n sharing P and Q folded into one, names in order."""
merged: list[Link] = []
folds: dict[tuple[str, str, bool], tuple[Link, list[Link]]] = {}
for link in links:
bios = link.bios_side.rstrip("/")
bios_is_target = link.bios_is_target
other = (link.link if bios_is_target else link.target).rstrip("/")
name = bios.rsplit("/", 1)[-1]
if link.names or other.rsplit("/", 1)[-1] != name:
merged.append(link)
continue
key = (bios.rsplit("/", 1)[0], other.rsplit("/", 1)[0], bios_is_target)
if key in folds:
folds[key][1].append(link)
else:
folds[key] = (link, [link])
merged.append(link)
folded: set[int] = set()
for (parent, other_parent, bios_is_target), (first, members) in folds.items():
if len(members) == 1:
continue
first.names = [m.bios_side.rstrip("/").rsplit("/", 1)[-1] for m in members]
first.bios_side = parent
bios_end, other_end = f"{parent}/<name>", f"{other_parent}/<name>"
first.target, first.link = (
(bios_end, other_end) if bios_is_target else (other_end, bios_end)
)
for member in members[1:]:
first.lines |= member.lines
folded.add(id(member))
return [link for link in merged if id(link) not in folded]
@dataclass
class Setting:
section: str
key: str
value: str
line: int
def settings(text: str, separator: str = "=") -> list[Setting]:
"""key = value lines under their [section], quotes around the value removed.
separator " " reads mednafen's "key value" form.
"""
found, section = [], ""
for number, line in enumerate(text.splitlines(), 1):
stripped = line.strip()
if not stripped or stripped[0] in "#;":
continue
header = re.fullmatch(r"\[([^\]]+)\]", stripped)
if header:
section = header.group(1)
continue
if separator == " ":
pair = re.fullmatch(r"(\S+)(?:\s+(.*))?", stripped)
else:
pair = re.fullmatch(r"([^=\s]+)\s*=\s*(.*)", stripped)
if pair is None:
continue
value = (pair.group(2) or "").strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
value = value[1:-1]
found.append(Setting(section, pair.group(1), value, number))
return found
def describe_keys(file: str, keys: tuple[tuple[str, str], ...]) -> str:
"""file [section] key, key [other] key, sections named where they change."""
described, section = file, None
for index, (key_section, key) in enumerate(keys):
if key_section and key_section != section:
described += f" [{key_section}] {key}"
section = key_section
else:
described += (" " if index == 0 else ", ") + key
return described
@dataclass
class Entry:
"""A directory entry: kind is "dir", "file" or "link"."""
kind: str
target: str = ""
class RepoTree:
"""Files of one repository revision, read on demand and kept."""
def __init__(
self,
read: Callable[[str], str],
listdir: Callable[[str], dict[str, Entry]],
) -> None:
self._read = read
self._listdir = listdir
self._texts: dict[str, str] = {}
self._dirs: dict[str, dict[str, Entry]] = {}
def read(self, path: str) -> str:
if path not in self._texts:
self._texts[path] = self._read(path)
return self._texts[path]
def listdir(self, path: str) -> dict[str, Entry]:
if path not in self._dirs:
self._dirs[path] = self._listdir(path)
return self._dirs[path]
def has(self, path: str) -> bool:
parent, _, name = path.rpartition("/")
try:
return name in self.listdir(parent)
except LauncherPatternMissing:
return False
@classmethod
def from_files(cls, files: dict[str, str], links: dict[str, str] | None = None) -> RepoTree:
"""A tree held in memory: file texts by path, symlink targets by path."""
links = links or {}
def read(path: str) -> str:
if path not in files:
raise LauncherPatternMissing(path, "file")
return files[path]
def listdir(path: str) -> dict[str, Entry]:
prefix = path.rstrip("/") + "/"
entries: dict[str, Entry] = {}
for name, target in links.items():
if name.startswith(prefix) and "/" not in name[len(prefix) :]:
entries[name[len(prefix) :]] = Entry("link", target)
for name in files:
if name.startswith(prefix):
rest = name[len(prefix) :]
head, _, tail = rest.partition("/")
entries.setdefault(head, Entry("dir" if tail else "file"))
if not entries:
raise LauncherPatternMissing(path, "directory")
return entries
return cls(read, listdir)
@classmethod
def github(cls, repo: str, rev: str) -> RepoTree:
"""A GitHub repository at one revision: raw files and the trees API."""
def read(path: str) -> str:
url = f"https://raw.githubusercontent.com/{repo}/{rev}/{path}"
return fetch_bytes(url).decode("utf-8-sig")
def listdir(path: str) -> dict[str, Entry]:
url = f"https://api.github.com/repos/{repo}/git/trees/{rev}:{path}"
try:
listing = json.loads(fetch_bytes(url, headers=github_headers()))
except ConnectionError as e:
raise LauncherPatternMissing(path, f"directory ({e})") from e
entries = {}
for item in listing.get("tree", []):
if item["mode"] == "120000":
entries[item["path"]] = Entry("link", read(f"{path}/{item['path']}").strip())
else:
entries[item["path"]] = Entry("dir" if item["type"] == "tree" else "file")
return entries
return cls(read, listdir)
+593
View File
@@ -0,0 +1,593 @@
"""ROCKNIX's launcher layer read from its packages.
Each fixture is a cut-down copy of the lines an extractor reads at
ROCKNIX/distribution 20261001 (c445081a59518f37d9776e5412dd7b14910696f7).
The records must carry the names in the order the code writes or tries them,
point at the fixture's own lines, name a device only where the shipped
configs differ, and a construct that is gone must stop the scrape.
"""
from __future__ import annotations
import sys
import textwrap
import unittest
from pathlib import Path
import yaml
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
from scraper import rocknix_launchers as rl # noqa: E402
from scraper.rocknix_scraper import launcher_revision # noqa: E402
from scraper.shell_launchers import LauncherPatternMissing, RepoTree # noqa: E402
SA = rl.STANDALONE
def text(block: str) -> str:
return textwrap.dedent(block).lstrip("\n")
def devices(*names: str) -> dict[str, str]:
return {f"{rl.DEVICES}/{name}/options": "" for name in names}
MELONDS_MK = text(
"""
makeinstall_target() {
mkdir -p ${INSTALL}/usr/config/melonDS
cp -a ${PKG_DIR}/config/${DEVICE}/* ${INSTALL}/usr/config/melonDS
}
"""
)
MELONDS_INI = text(
"""
ExternalBIOSEnable=0
BIOS9Path=/storage/roms/bios/bios9.bin
BIOS7Path=/storage/roms/bios/bios7.bin
FirmwarePath=/storage/roms/bios/firmware.bin
DSiBIOS9Path=/storage/roms/bios/dsi_bios9.bin
DSiBIOS7Path=/storage/roms/bios/dsi_bios7.bin
DSiFirmwarePath=/storage/roms/bios/dsi_firmware.bin
DSiNANDPath=/storage/roms/bios/dsi_nand.bin
"""
)
MELONDS_AMD64_INI = text(
"""
ExternalBIOSEnable=0
BIOS9Path=
BIOS7Path=
FirmwarePath=
DSiBIOS9Path=
DSiBIOS7Path=
DSiFirmwarePath=
DSiNANDPath=
"""
)
MELONDS_SH = text(
"""
CONTYPE=$(get_setting console_type "${PLATFORM}" "${GAME}")
DBOOT=$(get_setting direct_boot "${PLATFORM}" "${GAME}")
if [ "$PLATFORM" = "ndsiware" ]; then
sed -i '/^DirectBoot=/c\\DirectBoot=0' /storage/.config/melonDS/melonDS.ini
else
if [ "$DBOOT" = "0" ]; then
sed -i '/^DirectBoot=/c\\DirectBoot=0' /storage/.config/melonDS/melonDS.ini
sed -i '/^ExternalBIOSEnable=/c\\ExternalBIOSEnable=1' /storage/.config/melonDS/melonDS.ini
else
sed -i '/^ExternalBIOSEnable=/c\\ExternalBIOSEnable=0' /storage/.config/melonDS/melonDS.ini
fi
fi
"""
)
def melonds_tree() -> RepoTree:
pkg = f"{SA}/melonds-sa"
return RepoTree.from_files(
{
**devices("AMD64", "H700", "RK3576", "S922X"),
f"{pkg}/package.mk": MELONDS_MK,
f"{pkg}/scripts/start_melonds.sh": MELONDS_SH,
f"{pkg}/config/AMD64/melonDS.ini": MELONDS_AMD64_INI,
f"{pkg}/config/InputPlumber/melonDS.ini": MELONDS_INI,
f"{pkg}/config/S922X/melonDS.ini": MELONDS_INI,
},
links={f"{pkg}/config/RK3576": "InputPlumber"},
)
DSPERATE_MK = text(
"""
makeinstall_target() {
case ${DEVICE} in
RK3576|SM4450|SM6115|SM8250|SM8550|SM8650|SM8750) CONFIG="InputPlumber" ;;
*) CONFIG="${DEVICE}" ;;
esac
cp -a ${PKG_DIR}/config/${CONFIG}/* ${INSTALL}/usr/config/dsperate
}
"""
)
DSPERATE_INI = text(
"""
[paths]
bios9 = /storage/roms/bios/bios9.bin
bios7 = /storage/roms/bios/bios7.bin
firmware = /storage/roms/bios/firmware.bin
bios9i = /storage/roms/bios/dsi_bios9.bin
bios7i = /storage/roms/bios/dsi_bios7.bin
dsi_firmware = /storage/roms/bios/dsi_firmware.bin
dsi_nand = /storage/roms/bios/dsi_nand.bin
games = /storage/roms/nds
"""
)
AMIBERRY_SH = text(
"""
AMIBERRY_DIR="/storage/.config/amiberry"
AMIBERRY_CONFIG_DIR="${AMIBERRY_DIR}/conf"
BIOS_PATH="/storage/roms/bios"
AMIBERRY_SET_CONF() {
if [ `echo $1 | grep -E 'AGA|CD32' | wc -l` -eq 1 ]; then
cp ${AMIBERRY_CONFIG_DIR}/AmigaA1200-default.uae "${AMIBERRY_TMP_CONFIG}"
else
cp ${AMIBERRY_CONFIG_DIR}/AmigaA500-default.uae "${AMIBERRY_TMP_CONFIG}"
fi
sed -i "s#@BIOS_PATH@#${BIOS_PATH}#g" ${AMIBERRY_TMP_CONFIG}
}
"""
)
AMIBERRY_MK = text(
"""
makeinstall_target() {
mkdir -p ${INSTALL}/usr/config/amiberry
cp -a ${PKG_DIR}/config/* ${INSTALL}/usr/config/amiberry
ln -s /storage/roms/bios ${INSTALL}/usr/config/amiberry/kickstarts
ln -s /usr/share/libretro/autoconfig ${INSTALL}/usr/config/amiberry/controller
}
"""
)
MEDNAFEN_TEMPLATE = text(
"""
;Path to firmware directory.
filesys.path_firmware /storage/roms/bios/
;Path to PlayStation BIOS, Europe.
psx.bios_eu scph5502.bin
;Path to PlayStation BIOS, Japan.
psx.bios_jp scph5500.bin
;Enable BIOS sanity check.
psx.bios_sanity 1
;Path to PC Engine CD BIOS.
pce.cdbios syscard3.pce
"""
)
MEDNAFEN_SH = text(
"""
if [ ! -f "$MEDNAFEN_HOME/mednafen.cfg" ]
then
/usr/bin/bash /usr/bin/mednafen_gen_config.sh
fi
"""
)
YABASANSHIRO_SH = text(
"""
BIOS=""
USE_BIOS=$(get_setting use_hlebios "${PLATFORM}" "${GAME}")
if [ "$USE_BIOS" = 1 ]
then
for BIOS in saturn_bios.bin sega_101.bin mpr-17933.bin mpr-18811-mx.ic1 mpr-19367-mx.ic1 stvbios.zip
do
BIOS=$(find /storage/roms/bios -name ${BIOS} -print 2>/dev/null)
if [ ! -z "${BIOS}" ]
then
BIOS="-b ${BIOS}"
break
fi
done
fi
"""
)
DUCKSTATION_MK = text(
"""
makeinstall_target() {
cp -a ${PKG_DIR}/config/${DEVICE}/* ${INSTALL}/usr/config/duckstation
}
"""
)
DRASTIC_SH = text(
"""
for bios in nds_bios_arm9.bin nds_bios_arm7.bin
do
if [ ! -e "/storage/.config/drastic/system/${bios}" ]; then
if [ -e "/storage/roms/bios/${bios}" ]; then
ln -sf /storage/roms/bios/${bios} /storage/.config/drastic/system
fi
fi
done
"""
)
DOLPHIN_SH = text(
"""
CONF_DIR="/storage/.config/dolphin-emu"
ln -sf /storage/roms/savestates/gamecube ${CONF_DIR}/StateSaves
for REGION in EUR JAP USA
do
rm -rf "${CONF_DIR}/GC/${REGION}"
ln -sf "/storage/roms/bios/GC/${REGION}" "${CONF_DIR}/GC/${REGION}"
for SLOT in A B
do
MEM_CARD_FILE="MemoryCard${SLOT}.${REGION}.raw"
CONFIG_MEM_CARD="${CONF_DIR}/GC/${MEM_CARD_FILE}"
ROMS_BIOS_MEM_CARD="/storage/roms/bios/GC/${MEM_CARD_FILE}"
if [ -f "${ROMS_BIOS_MEM_CARD}" ]; then
ln -sf "${ROMS_BIOS_MEM_CARD}" "${CONFIG_MEM_CARD}"
elif [ -f "${CONFIG_MEM_CARD}" ]; then
ln -sf "${ROMS_BIOS_MEM_CARD}" "${CONFIG_MEM_CARD}"
fi
done
done
"""
)
CEMU_SH = text(
"""
CEMU_CONFIG_ROOT="/storage/.config/Cemu"
CEMU_HOME_CONFIG="${CEMU_CONFIG_ROOT}/share"
CEMU_HOME_ONLINE="${CEMU_HOME_CONFIG}/online"
CEMU_HOME_KEYS="${CEMU_HOME_CONFIG}/keys"
CEMU_BIOS="/storage/roms/bios/cemu"
if [ ! -L ${CEMU_HOME_ONLINE} ]; then
ln -sf ${CEMU_BIOS}/online ${CEMU_HOME_ONLINE}
fi
if [ ! -L ${CEMU_HOME_KEYS} ]; then
ln -sf ${CEMU_BIOS}/keys ${CEMU_HOME_KEYS}
fi
"""
)
RPCS3_SH = text(
"""
FOLDER_LINKS=("dev_flash" "dev_hdd0" "dev_hdd1" "custom_configs")
for FOLDER_LINK in "${FOLDER_LINKS[@]}"; do
TARGET_FOLDER="/storage/roms/bios/rpcs3/$FOLDER_LINK"
SOURCE_FOLDER="/storage/.config/rpcs3/$FOLDER_LINK"
rm -rf "$SOURCE_FOLDER"
ln -sf "$TARGET_FOLDER" "$SOURCE_FOLDER"
done
"""
)
FLYCAST_SH = text(
"""
if [ ! -d "/storage/roms/bios/dc" ]; then
mkdir -p "/storage/roms/bios/dc"
fi
ln -sf "/storage/roms/bios/dc" "/storage/roms/dreamcast/data"
"""
)
def by(records: list[dict], emulator: str) -> list[dict]:
return [r for r in records if r["emulator"] == emulator]
class ConfigsPerDevice(unittest.TestCase):
def test_a_device_appears_only_where_configs_differ(self):
records = [r.entry() for r in rl.melonds(melonds_tree())]
self.assertEqual([r["device"] for r in records], ["AMD64", "RK3576, S922X"])
empty, named = records
self.assertEqual(empty["names"], [])
self.assertTrue(empty["passes_to"].endswith("DSiNANDPath left empty"))
self.assertEqual(
named["names"],
[
"bios9.bin",
"bios7.bin",
"firmware.bin",
"dsi_bios9.bin",
"dsi_bios7.bin",
"dsi_firmware.bin",
"dsi_nand.bin",
],
)
self.assertEqual(named["search_dir"], "/storage/roms/bios")
def test_a_linked_device_directory_reads_its_target(self):
named = rl.melonds(melonds_tree())[1].entry()
self.assertIn(f"{SA}/melonds-sa/config/InputPlumber/melonDS.ini:2-8", named["source_ref"])
self.assertNotIn("config/RK3576", named["source_ref"])
def test_the_external_bios_switch_is_the_option(self):
named = rl.melonds(melonds_tree())[1].entry()
self.assertEqual(
named["depends_on_option"], "direct_boot = 0 sets ExternalBIOSEnable=1, otherwise 0"
)
self.assertIn(f"{SA}/melonds-sa/scripts/start_melonds.sh:2, 6-8", named["source_ref"])
def test_identical_configs_give_one_record_without_device(self):
pkg = f"{SA}/dsperate-sa"
tree = RepoTree.from_files(
{
**devices("H700", "RK3576", "S922X"),
f"{pkg}/package.mk": DSPERATE_MK,
f"{pkg}/scripts/start_dsperate.sh": '${EMUPERF} /usr/bin/dsperate --config "${DSPERATE_INI}" "${ROM}"\n',
f"{pkg}/config/H700/dsperate.ini": DSPERATE_INI,
f"{pkg}/config/InputPlumber/dsperate.ini": DSPERATE_INI,
f"{pkg}/config/S922X/dsperate.ini": DSPERATE_INI,
}
)
(record,) = [r.entry() for r in rl.dsperate(tree)]
self.assertNotIn("device", record)
self.assertEqual(
record["passes_to"],
"dsperate.ini [paths] bios9, bios7, firmware, bios9i, bios7i, dsi_firmware, dsi_nand (--config)",
)
# The case statement sends RK3576 to InputPlumber.
self.assertIn(f"{pkg}/config/InputPlumber/dsperate.ini:2-8", record["source_ref"])
self.assertIn(f"{pkg}/package.mk:3-4", record["source_ref"])
def test_an_absent_key_and_an_empty_one_hand_over_the_same(self):
pkg = f"{SA}/duckstation-sa"
tree = RepoTree.from_files(
{
**devices("AMD64", "S922X"),
f"{pkg}/package.mk": DUCKSTATION_MK,
f"{pkg}/config/AMD64/settings.ini": "[BIOS]\nSearchDirectory = /storage/roms/bios\n",
f"{pkg}/config/S922X/settings.ini": text(
"""
[BIOS]
SearchDirectory = /storage/roms/bios
PathNTSCU =
PathNTSCJ =
PathPAL =
"""
),
}
)
(record,) = [r.entry() for r in rl.duckstation(tree)]
self.assertEqual(record["kind"], "scans_dir")
self.assertEqual(record["search_dir"], "/storage/roms/bios")
self.assertNotIn("device", record)
def test_a_bios_written_into_a_path_key_is_an_unknown_shape(self):
pkg = f"{SA}/duckstation-sa"
tree = RepoTree.from_files(
{
**devices("S922X"),
f"{pkg}/package.mk": DUCKSTATION_MK,
f"{pkg}/config/S922X/settings.ini": "[BIOS]\nSearchDirectory = /storage/roms/bios\nPathNTSCU = scph5501.bin\n",
}
)
with self.assertRaises(LauncherPatternMissing):
rl.duckstation(tree)
def test_a_package_without_a_device_copy_raises(self):
pkg = f"{SA}/duckstation-sa"
tree = RepoTree.from_files(
{
**devices("S922X"),
f"{pkg}/package.mk": "makeinstall_target() {\n true\n}\n",
f"{pkg}/config/S922X/settings.ini": "[BIOS]\nSearchDirectory = /storage/roms/bios\n",
}
)
with self.assertRaises(LauncherPatternMissing):
rl.duckstation(tree)
class NamesTheConfigsFix(unittest.TestCase):
def test_amiberry_machine_follows_the_game_name(self):
pkg = f"{SA}/amiberry"
uae = "amiberry.rom_path=/storage/.config/amiberry/kickstarts/\nkickstart_rom_file=@BIOS_PATH@/{}\nkickstart_ext_rom_file=\n"
tree = RepoTree.from_files(
{
f"{pkg}/scripts/start_amiberry.sh": AMIBERRY_SH,
f"{pkg}/package.mk": AMIBERRY_MK,
f"{pkg}/config/conf/AmigaA1200-default.uae": uae.format("kick40068.A1200.rom"),
f"{pkg}/config/conf/AmigaA500-default.uae": uae.format("kick34005.A500.rom"),
}
)
a1200, a500, link = (r.entry() for r in rl.amiberry(tree))
self.assertEqual((a1200["slot"], a1200["names"]), ("A1200", ["kick40068.A1200.rom"]))
self.assertEqual(a1200["depends_on_option"], "game name matches AGA|CD32")
self.assertEqual((a500["slot"], a500["names"]), ("A500", ["kick34005.A500.rom"]))
self.assertEqual(a500["depends_on_option"], "game name does not match AGA|CD32")
self.assertEqual(a500["search_dir"], "/storage/roms/bios")
self.assertEqual(
a1200["source_ref"],
f"{pkg}/scripts/start_amiberry.sh:3-10, {pkg}/config/conf/AmigaA1200-default.uae:2",
)
self.assertEqual(link["kind"], "links")
self.assertEqual(
link["passes_to"], "symlink /usr/config/amiberry/kickstarts -> /storage/roms/bios"
)
self.assertEqual(link["source_ref"], f"{pkg}/package.mk:4")
def test_mednafen_one_record_per_module(self):
pkg = f"{SA}/mednafen"
tree = RepoTree.from_files(
{
f"{pkg}/config/common/mednafen.template": MEDNAFEN_TEMPLATE,
f"{pkg}/scripts/start_mednafen.sh": MEDNAFEN_SH,
}
)
psx, pce = (r.entry() for r in rl.mednafen(tree))
self.assertEqual((psx["slot"], psx["names"]), ("psx", ["scph5502.bin", "scph5500.bin"]))
self.assertEqual(psx["passes_to"], "mednafen.cfg psx.bios_eu, psx.bios_jp")
self.assertEqual(pce["names"], ["syscard3.pce"])
self.assertEqual(psx["search_dir"], "/storage/roms/bios/")
self.assertEqual(
psx["source_ref"],
f"{pkg}/config/common/mednafen.template:2, 5, 8, {pkg}/scripts/start_mednafen.sh:1-3",
)
def test_mednafen_without_a_firmware_directory_raises(self):
pkg = f"{SA}/mednafen"
tree = RepoTree.from_files(
{
f"{pkg}/config/common/mednafen.template": "psx.bios_eu scph5502.bin\n",
f"{pkg}/scripts/start_mednafen.sh": MEDNAFEN_SH,
}
)
with self.assertRaises(LauncherPatternMissing):
rl.mednafen(tree)
class SearchedNames(unittest.TestCase):
def tree(self, script: str) -> RepoTree:
return RepoTree.from_files({f"{SA}/yabasanshiro-sa/scripts/start_yabasanshiro.sh": script})
def test_yabasanshiro_tries_its_names_in_order(self):
(record,) = [r.entry() for r in rl.yabasanshiro(self.tree(YABASANSHIRO_SH))]
self.assertEqual(record["kind"], "names_searched")
self.assertEqual(
record["names"],
[
"saturn_bios.bin",
"sega_101.bin",
"mpr-17933.bin",
"mpr-18811-mx.ic1",
"mpr-19367-mx.ic1",
"stvbios.zip",
],
)
self.assertEqual(record["passes_to"], "yabasanshiro -b <first path found>")
self.assertEqual(record["depends_on_option"], "use_hlebios = 1 (game, then platform)")
self.assertIs(record["case_insensitive"], False)
self.assertEqual(
record["source_ref"], f"{SA}/yabasanshiro-sa/scripts/start_yabasanshiro.sh:2-13"
)
def test_a_search_no_longer_written_as_a_loop_raises(self):
script = YABASANSHIRO_SH.replace("for BIOS in", "BIOS_LIST=").replace(" do\n", "")
with self.assertRaises(LauncherPatternMissing):
rl.yabasanshiro(self.tree(script))
class Links(unittest.TestCase):
def all_scripts(self) -> dict[str, str]:
return {
"drastic": DRASTIC_SH,
"dolphin": DOLPHIN_SH,
"flycast": FLYCAST_SH,
"cemu": CEMU_SH,
"rpcs3": RPCS3_SH,
}
def tree(self) -> RepoTree:
scripts = self.all_scripts()
return RepoTree.from_files({p: scripts[e] for e, p in rl.LINK_SCRIPTS})
def emulator(self, name: str) -> list[dict]:
return by([r.entry() for r in rl.links(self.tree())], name)
def test_a_loop_names_each_file_and_links_into_a_directory(self):
(record,) = self.emulator("drastic")
self.assertEqual(record["names"], ["nds_bios_arm9.bin", "nds_bios_arm7.bin"])
self.assertEqual(record["search_dir"], "/storage/roms/bios")
self.assertEqual(
record["passes_to"],
"symlink /storage/.config/drastic/system/<name> -> /storage/roms/bios/<name>",
)
# The link waits for an exact-case -e test.
self.assertIs(record["case_insensitive"], False)
self.assertEqual(record["source_ref"], f"{rl.LINK_SCRIPTS[0][1]}:1, 4-5")
def test_a_nested_loop_is_a_placeholder_and_its_values(self):
regions, cards = self.emulator("dolphin")
self.assertEqual(regions["names"], ["EUR", "JAP", "USA"])
self.assertEqual(regions["search_dir"], "/storage/roms/bios/GC")
self.assertEqual(cards["names"], ["MemoryCard{SLOT}.{REGION}.raw"])
self.assertEqual(cards["expands"], {"REGION": ["EUR", "JAP", "USA"], "SLOT": ["A", "B"]})
def test_a_link_outside_the_bios_directory_is_not_a_record(self):
for record in self.emulator("dolphin"):
self.assertNotIn("StateSaves", record["passes_to"])
def test_one_by_one_links_of_one_layout_are_one_record(self):
(record,) = self.emulator("cemu")
self.assertEqual(record["names"], ["online", "keys"])
self.assertEqual(
record["passes_to"],
"symlink /storage/.config/Cemu/share/<name> -> /storage/roms/bios/cemu/<name>",
)
def test_an_array_loop_takes_the_array(self):
(record,) = self.emulator("rpcs3")
self.assertEqual(record["names"], ["dev_flash", "dev_hdd0", "dev_hdd1", "custom_configs"])
self.assertEqual(record["search_dir"], "/storage/roms/bios/rpcs3")
self.assertEqual(record["source_ref"], f"{dict(rl.LINK_SCRIPTS)['rpcs3']}:1-6")
def test_a_directory_link_has_no_names(self):
(record,) = self.emulator("flycast")
self.assertEqual(record["names"], [])
self.assertEqual(
record["passes_to"], "symlink /storage/roms/dreamcast/data -> /storage/roms/bios/dc"
)
def test_a_script_without_a_bios_link_raises(self):
scripts = self.all_scripts()
scripts["flycast"] = 'ln -sf "/storage/.config/flycast" "/storage/.local/share/flycast"\n'
tree = RepoTree.from_files({p: scripts[e] for e, p in rl.LINK_SCRIPTS})
with self.assertRaises(LauncherPatternMissing):
rl.links(tree)
class Revision(unittest.TestCase):
def test_the_revision_is_the_one_the_bios_list_is_read_at(self):
url = (
"https://raw.githubusercontent.com/ROCKNIX/distribution/20261001/"
"projects/ROCKNIX/packages/rocknix/sources/scripts/rocknix-systems"
)
self.assertEqual(launcher_revision(url), "20261001")
def test_a_url_without_a_revision_raises(self):
with self.assertRaises(ValueError):
launcher_revision("https://rocknix.org/")
class PlatformRecords(unittest.TestCase):
"""The launchers platforms/rocknix.yml carries, as the scraper wrote them."""
@classmethod
def setUpClass(cls) -> None:
cls.config = yaml.safe_load((REPO_ROOT / "platforms" / "rocknix.yml").read_text())
def test_every_emulator_is_one_rocknix_packages(self):
cores = set(self.config["cores"])
self.assertEqual(sorted({r["emulator"] for r in self.config["launchers"]} - cores), [])
def test_every_reference_is_in_the_rocknix_project(self):
for record in self.config["launchers"]:
for part in record["source_ref"].split(", "):
if ":" in part:
self.assertTrue(part.startswith(f"{rl.ROCKNIX}/"), record)
def test_a_device_is_named_only_beside_another_device_record(self):
with_device: dict[tuple[str, str], int] = {}
for record in self.config["launchers"]:
if "device" in record:
key = (record["emulator"], record["kind"])
with_device[key] = with_device.get(key, 0) + 1
self.assertTrue(with_device)
self.assertEqual({k: n for k, n in with_device.items() if n < 2}, {})
if __name__ == "__main__":
unittest.main()
+21 -3
View File
@@ -234,11 +234,29 @@ launchers:
| `fails_when_missing` | `false`, or the message the launcher raises |
| `fails_scope` | `emulator` when the failure needs every record of the emulator to come up empty |
| `depends_on_option` | Options or conditions that decide whether the record applies |
| `device` | Devices whose shipped config the record reads, set only where a package ships configs that differ between devices |
| `source_ref` | `path:lines` at the revision `source` pins |
Each record is read with `ast` from the generator. An extractor that no longer
recognises the shape of the code raises: an empty list would read as a
launcher that looks for nothing.
Batocera's generators are read with `ast`, RetroBat's C# generators as text.
ROCKNIX and EmuDeck launch from shell scripts and configs shipped with each
package, so `scripts/scraper/shell_launchers.py` reads assignments, `for`
loops, `ln -s` commands and `key = value` lines. A ROCKNIX package can ship
one config per device: the configs are grouped by the value read, and
`device` appears only when there is more than one group. melonDS writes its
seven BIOS paths on every device but AMD64, where the keys are left empty:
```yaml
- emulator: melonds
build: standalone
kind: fixed_name
names: []
passes_to: melonDS.ini BIOS9Path, BIOS7Path, FirmwarePath, DSiBIOS9Path,
DSiBIOS7Path, DSiFirmwarePath, DSiNANDPath left empty
device: AMD64
```
An extractor that no longer recognises the shape of the code raises: an
empty list would read as a launcher that looks for nothing.
### Inheritance