chore: merge the launcher search paths

This commit is contained in:
Abdessamad Derraz committed 2026-11-10 13:12:22 +01:00
commit c69b31cfc0
23 files changed
+2373 -161

No files matched your search

+16 -3
View File
@@ -1646,12 +1646,15 @@ def resolve_local_file(
layout = file_entry.get("layout_path")
if layout and layout != dest_hint:
plain = {k: v for k, v in file_entry.items() if k != "layout_path"}
# A destination outside the BIOS tree says nothing of the collection:
# EmuELEC's roms/mame/konamigv.zip is not bios/Arcade/MAME's set.
outside = file_entry.get("unprefixed") or file_entry.get("root")
answers = [
resolve_local_file(
plain, db, zip_contents, dest_hint=hint, _depth=_depth,
data_dir_registry=data_dir_registry,
)
for hint in (dest_hint, str(layout))
for hint in ((str(layout),) if outside else (dest_hint, str(layout)))
]
for found in answers:
if found[0] is not None and (
@@ -3098,12 +3101,14 @@ def standalone_layout_path(entry: dict, profile: dict) -> str | None:
``standalone_path`` when the entry has one. A profile with no libretro
build has one layout, so its ``path`` is that layout: UAE4All2 reads
kickstarts/kick13.rom below its working directory.
kickstarts/kick13.rom below its working directory. So does an entry only
the standalone build reads (``mode: standalone``), whose destination is
its ``path``: Dolphin's Load/Firmware/rtl_bt/ under its User directory.
"""
own = entry.get("standalone_path")
if own:
return own
if "libretro" in str(profile.get("type", "")):
if "libretro" in str(profile.get("type", "")) and entry.get("mode") != "standalone":
return None
return entry.get("path") or None
@@ -3230,6 +3235,14 @@ def platform_layouts(
tree, path = placed
moved = {**f, "standalone_path": path,
"layout_path": standalone_layout_path(f, profile)}
if f.get("load_from"):
# The build's own root no longer applies: RetroDECK hands
# Cemu's MLC, read from its data directory, to bios/cemu.
load_from = f["load_from"]
moved["load_from"] = {
**(load_from if isinstance(load_from, dict) else {"libretro": load_from}),
"standalone": "system_dir",
}
if tree == "install":
# Below the roms root inside the install tree (AmberELEC's
# /storage/roms): a plain dest the platform prefix skips.
+74 -16
View File
@@ -312,6 +312,11 @@ def launcher_places(config: dict) -> list[LauncherPlace]:
# component scripts $bios_path.
BIOS_ROOT_VARIABLES = ("$biosdir", "$home/RetroPie/BIOS", "${biosPath}", "$bios_path")
# The name a platform's launch code gives its roms root: RetroDECK's
# component scripts write $roms_path (retrodeck_launchers), the setting whose
# default is the roms root (config/retrodeck/retrodeck.json:5).
ROMS_ROOT_VARIABLES = ("$roms_path", "${roms_path}")
# The kinds that hand an emulator a whole directory. A scans_dir record sets
# the directory the emulator picks its file from: ROCKNIX writes ARMSX2's
# [Folders] Bios, EmuDeck PCSX2's.
@@ -341,20 +346,29 @@ INSTALL_TREE = "install"
# A folder of neither tree, which no pack and no installer can fill.
OUTSIDE_TREE = "outside"
Place = tuple[str, str]
# The binding key of a search list standing for the whole layout: the
# directories hold the layout's files themselves (XRoar's ROMs, MAME's sets),
# never its subfolders.
SEARCHED_FILES = "*"
_HOME = re.compile(r"^(?:\$\{HOME\}|\$HOME|\$home|~)(?=/|$)")
# Flatpak sets XDG_DATA_HOME to ~/.var/app/<id>/data inside the sandbox.
_FLATPAK_DATA = re.compile(r"^~/\.var/app/[^/]+/data(?=/|$)")
def normalize_handle(token: str) -> str:
def normalize_handle(token: str, home: str = "") -> str:
"""One spelling for a path, setting or function a launcher record names.
``$home``, ``$HOME``, ``${HOME}`` and ``~`` are one directory, and a
Flatpak's data directory is its ``$XDG_DATA_HOME``.
Flatpak's data directory is its ``$XDG_DATA_HOME``. ``home`` is the
platform's home directory spelled out (``home_directory``): EmuELEC
links /storage/.local/share/flycast, Flycast's ~/.local/share/flycast.
"""
token = token.strip().strip("\"'").rstrip(",;").rstrip("/")
token = token.removesuffix("()")
home = home.rstrip("/")
if home and _below(token, home) is not None:
token = "~" + token[len(home):]
token = _HOME.sub("~", token)
return _FLATPAK_DATA.sub("$XDG_DATA_HOME", token)
@@ -438,13 +452,29 @@ def native_systems(config: dict) -> dict[str, str]:
return found
def _listed_directories(raw: str) -> list[str] | None:
"""The directories of a colon-separated search path, None for one directory.
# How the launcher extractors spell the folder holding the launched game:
# RetroBat's Path.GetDirectoryName(rom) (retrobat_launchers._KNOWN), KNULLI's
# os.path.dirname(rom) and its software list folder (mameGenerator.py:113-116).
GAME_FOLDERS = ("<rom directory>", "<romDirname>", "<softDir>")
XRoar's rompath is such a list (RetroDECK writes four directories), each
entry a path or a variable spelling one. A Windows drive letter or a
backslash means a single path.
def _listed_directories(raw: str) -> list[str] | None:
"""The directories of a search path, None for one directory.
XRoar's rompath is a colon-separated list (RetroDECK writes four
directories), each entry a path or a variable spelling one. A Windows
drive letter or a backslash means a single path. MAME splits its search
paths on semicolons whatever the system (src/emu/fileio.cpp:53-58):
RetroBat passes `-rp bios;<rom directory>`, KNULLI `-rompath
<romDirname>;/userdata/bios/mame;/userdata/bios`. The game's own folder
is no folder a pack fills, so it gives no directory.
"""
if ";" in raw:
parts = [p.strip() for p in raw.split(";") if p.strip()]
kept = [p for p in parts if p not in GAME_FOLDERS]
if kept and all(not re.search(r"[\s,<>]", p) for p in kept):
return kept
return None
if ":" not in raw or "\\" in raw or re.match(r"^[A-Za-z]:", raw):
return None
parts = [p.strip() for p in raw.split(":")]
@@ -464,6 +494,7 @@ def handed_directories(config: dict) -> list[HandedDirectory]:
holds <system> gives one directory per system the platform declares.
"""
base = str(config.get("base_destination") or "bios")
home = str(config.get("home_directory") or "")
natives = native_systems(config)
declared = {d.lower() for d in declared_destinations(config)}
found: list[HandedDirectory] = []
@@ -490,7 +521,7 @@ def handed_directories(config: dict) -> list[HandedDirectory]:
if any(re.search(r"[\s,:;]", part) for part in parts):
continue
for part in parts:
directory = normalize_handle(part)
directory = normalize_handle(part, home)
if passes_to.startswith(WORKING_DIRECTORY_PASSED):
# The program runs from the directory: what its code
# reads through getcwd(), or relative to it, is below it.
@@ -499,7 +530,7 @@ def handed_directories(config: dict) -> list[HandedDirectory]:
handles = frozenset(
handle
for token in passes_to.split()
if (handle := normalize_handle(token)) and handle != directory
if (handle := normalize_handle(token, home)) and handle != directory
) | ini_keys(passes_to)
folder = _folder_in_bios(part, base)
for emulator in (e.strip() for e in str(record.get("emulator", "")).split(",")):
@@ -565,8 +596,15 @@ def folder_in_roms(directory: str, config: dict) -> str | None:
(``roms_root.path``): ROCKNIX's /storage/roms, which holds its BIOS
folder and the folders its launchers run programs from.
"""
from common import ROMS_ROOT # noqa: PLC0415
from common import ROMS_ROOT, outer_layout # noqa: PLC0415
spelled = directory.strip().rstrip("/")
for variable in ROMS_ROOT_VARIABLES:
if outer_layout(config, ROMS_ROOT) and (
spelled == variable or spelled.startswith(variable + "/")
):
rest = spelled[len(variable) + 1:]
return None if re.search(r"[\s<>$:;]", rest) else rest
return folder_in_root(directory, config, ROMS_ROOT)
@@ -716,7 +754,13 @@ def _bind(
one after the other: openMSX's filepool, XRoar's rompath. A file in any
of them is read, so the pool gives one: the directory the platform's own
list already fills (Batocera's bios/openmsx beside bios/Machines), else
the first of the BIOS tree, else the first a pack can fill.
the first of the BIOS tree, else the first a pack can fill. A list that
already holds the folder the layout's files take without the launcher
keeps them there: KNULLI starts MAME with -rompath
<romDirname>;/userdata/bios/mame;/userdata/bios, and the sets at the BIOS
root stay. A pool standing for the whole layout holds its files
directly, not its folders (SEARCHED_FILES): MAME reads hash/ and
samples/ through other settings.
"""
found: dict[str, set[Place]] = {}
pooled: dict[str, list[tuple[bool, Place]]] = {}
@@ -750,11 +794,14 @@ def _bind(
for rest, place in places or [("", (OUTSIDE_TREE, ""))]:
found.setdefault(rest, set()).add(place)
for sub, entries in pooled.items():
# The directory the platform's own list fills, else the first of the
# BIOS tree, else the first a pack fills.
# The folder the files already take, else the directory the
# platform's own list fills, else the first of the BIOS tree, else
# the first a pack fills.
unmoved = [p for _held, p in entries if p == (BIOS_TREE, sub)]
filled = [p for held, p in entries if held]
in_bios = [p for _held, p in entries if p[0] == BIOS_TREE]
found.setdefault(sub, set()).add((filled or in_bios or [p for _h, p in entries])[0])
chosen = (unmoved or filled or in_bios or [p for _h, p in entries])[0]
found.setdefault(sub or SEARCHED_FILES, set()).add(chosen)
return found
@@ -844,9 +891,15 @@ def bound_path(path: str, bindings: dict[str, Place]) -> Place | None:
The longest subfolder that contains the path wins: a link of
kickstarts/ places kickstarts/kick13.rom even when the whole layout is
bound elsewhere.
bound elsewhere. SEARCHED_FILES covers the files at the root of the
layout alone.
"""
for sub in sorted(bindings, key=len, reverse=True):
if sub == SEARCHED_FILES:
if "/" in path:
continue
root, folder = bindings[sub]
return (root, f"{folder}/{path}" if folder else path)
if sub and not path.startswith(sub + "/"):
continue
rest = path[len(sub) + 1:] if sub else path
@@ -1087,7 +1140,9 @@ class LauncherLayer:
# core asked for it: the pack puts a file there either way.
reported_at: dict[str, dict] = {}
for _emulator, _standalone, _entry, report in candidates:
if report.get("in_repo"):
# A file the layout reads outside the BIOS tree fills no
# destination of it.
if report.get("in_repo") and not report.get("unplaced"):
destination = sanitize_pack_path(report.get("path") or report.get("name", ""))
reported_at.setdefault(destination, report)
by_layout: dict[tuple[str, bool], list[tuple[dict, dict]]] = {}
@@ -1131,6 +1186,9 @@ class LauncherLayer:
continue
if isinstance(filled, dict):
added.append(filled)
# The launcher hands the emulator that file from the BIOS
# tree: read outside it no more.
settled.update(id(report) for _c, report in matched if report.get("unplaced"))
if place.stands_for_default:
# The emulator opens the launcher's file instead of its
# own name, so a copy under that name is read by nobody:
+9 -2
View File
@@ -1462,7 +1462,7 @@ def _collect_emulator_extras(
# covered elsewhere belongs to another program's file.
# Nor is one placed outside the BIOS tree by the layout.
if (fname not in covered_names or f.get("unsourceable") or f.get("unplaced")
or f.get("root") in OUTER_ROOTS or f.get("unprefixed")):
or f.get("root") in OUTER_ROOTS):
continue
# Installed with the emulator or by the platform's package,
# as the first pass already decided for the same entry.
@@ -1492,7 +1492,12 @@ def _collect_emulator_extras(
# of the directory core files go to, which the platform may not
# declare (Recalbox has its System Cards under pcenginecd/ only,
# geargrafx reads them from the root).
full_dest = prefixed_destination(dest, extras_prefix)
# A path of the install tree outside the BIOS folder keeps no
# prefix: EmuELEC's dolphin/dolphin-emu/GBA/gba_bios.bin.
full_dest = (
sanitize_pack_path(dest) if f.get("unprefixed")
else prefixed_destination(dest, extras_prefix)
)
if full_dest in seen_dests and full_dest not in taken:
continue
if full_dest in taken and full_dest not in seen and fname not in platform_names:
@@ -1538,6 +1543,8 @@ def _collect_emulator_extras(
extra["contents"] = f["contents"]
if f.get("system_dir"):
extra["system_dir"] = f["system_dir"]
if f.get("unprefixed"):
extra["unprefixed"] = True
extra["claim_order"] = (2, 0, emu_name)
if _take(extra, full_dest, seen_dests, taken):
extras.append(extra)
+3 -1
View File
@@ -37,11 +37,13 @@ class Launcher:
device: str | None = None
# The machines the platform offers under the record's systems.
machines: list[str] | None = None
# The platform systems the record is written for, none for every one.
systems: list[str] | None = None
def entry(self) -> dict[str, Any]:
"""The platform YAML form, in a fixed key order, unset keys left out."""
order = (
"emulator", "build", "kind", "slot", "names", "expands",
"emulator", "systems", "build", "kind", "slot", "names", "expands",
"case_insensitive", "search_dir", "passes_to",
"fails_when_missing", "fails_scope", "depends_on_option",
"device", "machines", "source_ref",
+49 -21
View File
@@ -38,6 +38,8 @@ EMULATORS = f"{SX05RE}/emulators"
LIBRETRO = f"{SX05RE}/libretro"
RETROARCH_MK = f"{SX05RE}/libretro_base/retroarch/package.mk"
BIOS_ROOT = "/storage/roms/bios"
# The roms root that holds it (packages/sx05re/libretro_base/retroarch/package.mk:112,117).
ROMS_ROOT = "/storage/roms"
_EMU_TEST = re.compile(r'\$\{?EMU\}?"?\s*==?\s*"([^"]+)"')
_ARM = re.compile(r'^\s*((?:"[^"]+"\|?)+)\)\s*$')
@@ -487,8 +489,18 @@ def hatari(tree: RepoTree, runs: Runs) -> list[Launcher]:
]
_ADVMAME_SYSTEM = re.compile(r'^\s*if \[\[ "\$\{1\}" = "([\w-]+)" \]\]; then$')
_ADVMAME_SED = re.compile(r'^\s*sed -i "s\|([^|]+)\|([^|]+)\|g" \$\{CONFIG_DIR\}/advmame\.rc$')
def advancemame(tree: RepoTree, runs: Runs) -> list[Launcher]:
"""Folder keys of the advmame.rc package.mk installs, one per device branch."""
"""Folder keys of the advmame.rc package.mk installs, one per device branch.
A key under the roms root is one too: dir_rom is /storage/roms/mame. The
start script rewrites the config for the system it is launched for
(advmame.sh:26-30): /roms/mame becomes /roms/arcade for arcade, so that
system reads its own record.
"""
package = f"{EMULATORS}/advancemame"
mk = _read(tree, f"{package}/package.mk")
installs = [
@@ -501,8 +513,8 @@ def advancemame(tree: RepoTree, runs: Runs) -> list[Launcher]:
by_value: dict[tuple[tuple[str, str], ...], list[tuple[str, set[int], int]]] = {}
for mk_line, file in installs:
path = f"{package}/config/{file}"
keys = [s for s in settings(tree.read(path), separator=" ") if _under(s.value)]
if not keys:
keys = [s for s in settings(tree.read(path), separator=" ") if _under(s.value, ROMS_ROOT)]
if not any(_under(s.value) for s in keys):
raise LauncherPatternMissing(path, f"key naming a folder under {BIOS_ROOT}")
value = tuple((s.key, s.value) for s in keys)
by_value.setdefault(value, []).append((path, {s.line for s in keys}, mk_line))
@@ -511,25 +523,41 @@ def advancemame(tree: RepoTree, runs: Runs) -> list[Launcher]:
((value, configs),) = by_value.items()
script = _read(tree, f"{package}/bin/advmame.sh")
run_line, _ = script.find(r"^\s*(?:\w+=\S+ )*advmame ", "advmame start")
test_line, test = script.find(_ADVMAME_SYSTEM.pattern, "system test", end=run_line)
sed_line, sed = script.find(_ADVMAME_SED.pattern, "system rewrite", test_line + 1, test_line + 1)
name = posixpath.basename(script.path)
refs = [
runs.ref(name),
*((path, lines) for path, lines, _ in configs),
(mk.path, {mk_line for _, _, mk_line in configs}),
(script.path, {run_line}),
]
return [
Launcher(
emulator=runs.emulator(name),
build="standalone",
kind="passes_path",
names=[],
source_ref=source_ref(refs),
search_dir=folder,
passes_to=f"advmame.rc {key}",
)
for key, folder in value
]
def refs(*lines: int) -> str:
return source_ref([
runs.ref(name),
*((path, lines) for path, lines, _ in configs),
(mk.path, {mk_line for _, _, mk_line in configs}),
(script.path, {run_line, *lines}),
])
found = []
for key, folder in value:
rewritten = folder.replace(sed.group(1), sed.group(2))
if rewritten != folder:
# Every system but the one the script rewrites the key for, and
# that system with the rewritten folder.
found.append(Launcher(
emulator=runs.emulator(name), build="standalone", kind="passes_path", names=[],
source_ref=refs(test_line, sed_line), search_dir=folder,
passes_to=f"advmame.rc {key}",
depends_on_option=f"system != {test.group(1)}",
))
found.append(Launcher(
emulator=runs.emulator(name), build="standalone", kind="passes_path", names=[],
source_ref=refs(test_line, sed_line), search_dir=rewritten,
passes_to=f"advmame.rc {key}", systems=[test.group(1)],
))
continue
found.append(Launcher(
emulator=runs.emulator(name), build="standalone", kind="passes_path", names=[],
source_ref=refs(), search_dir=folder, passes_to=f"advmame.rc {key}",
))
return found
def mame(tree: RepoTree, runs: Runs) -> list[Launcher]:
+41 -1
View File
@@ -3336,7 +3336,11 @@ def mame(sources: dict[str, Source]) -> list[Record]:
raise LauncherPatternMissing(own.path, "MAME folder arguments")
records = _covered(
own,
[_with_conditions(record, conditions) for record, conditions in found],
[
split
for record, conditions in found
for split in _per_emulator(own, _with_conditions(record, conditions))
],
# The bezel artwork folder is read by the launcher, not passed.
skip=(_BIOS_GUARD, r"ConfigureBezels\("),
)
@@ -3424,6 +3428,42 @@ def xroar(sources: dict[str, Source]) -> list[Record]:
return _covered(src, records)
_EMULATOR_BRANCH = re.compile(r"([^\s;<>]+) when (\w+), else ([^\s;<>]+)")
def _per_emulator(src: Source, record: Record) -> list[Record]:
"""A directory chosen by the emulator the generator runs, one record each.
Mame64.Generator.cs sets `bool hbmame = emulator == "hbmame";` and passes
`-sp bios/hbmame/samples` for it, bios/mame/samples for mame64. A choice
on any other condition is left as it reads.
"""
rendered = str(record.get("search_dir", ""))
branch = _EMULATOR_BRANCH.fullmatch(rendered)
if branch is None:
return [record]
chosen, flag, default = branch.groups()
declared = re.search(rf'bool {flag} = emulator == "([\w.-]+)";', _code(src))
if declared is None:
return [record]
ref = str(record["source_ref"])
if not ref.startswith(src.path + ":"):
raise LauncherPatternMissing(src.path, f"source of the directory choice {rendered!r}")
spans = [
(int(first), int(last or first))
for first, _, last in (s.partition("-") for s in ref[len(src.path) + 1:].split(", "))
]
spans.append(src.lines(declared.start(), declared.end()))
prefix = str(record["passes_to"]).removesuffix(rendered).strip()
out = []
for emulator, directory in ((declared.group(1), chosen), (record["emulator"], default)):
split = {k: v for k, v in record.items() if k != "source_ref"}
split.update(emulator=emulator, search_dir=directory, passes_to=f"{prefix} {directory}")
split["source_ref"] = _path_parts(src, *spans)
out.append(split)
return out
_SYSTEM_BRANCH = re.compile(r'(\S+), or (\S+) when system == "([\w.-]+)"')
+25 -8
View File
@@ -30,6 +30,10 @@ except ImportError:
BIOS = "$bios_path"
PLACEHOLDER = "RETRODECKBIOSDIR"
# The roms folder in the same presets: GZDoom's IWADSearch.Directories holds
# RETRODECKROMSDIR/doom (gzdoom/rd_assets/rd_config/gzdoom.ini:9), which
# gzdoom/component_prepare.sh:27 rewrites to $roms_path.
ROMS_PLACEHOLDER = "RETRODECKROMSDIR"
CORE_OPTIONS = "retroarch/rd_assets/rd_config/retroarch-core-options.cfg"
# The component that runs libretro cores.
LIBRETRO_COMPONENT = "retroarch"
@@ -282,7 +286,11 @@ class Component:
)
def _preset(self, script: Script, number: int, words: list[str]) -> None:
"""`sed -i "s#RETRODECKBIOSDIR#${bios_path}#g" "$cfg"` on a preset copied just above."""
"""`sed -i "s#RETRODECKBIOSDIR#${bios_path}#g" "$cfg"` on a preset copied just above.
Each line naming the BIOS or the roms folder is a record: a search
list can mix them (XRoar's rompath, GZDoom's IWAD directories).
"""
target = words[-1]
substitutions = self._substitutions(script, target)
copied = None
@@ -302,17 +310,25 @@ class Component:
stripped = line.strip()
if re.fullmatch(r"\[[^\]]+\]", stripped):
section = stripped
if PLACEHOLDER not in stripped or stripped.startswith("#"):
if stripped.startswith("#") or not any(
p in stripped for p in (PLACEHOLDER, ROMS_PLACEHOLDER)
):
continue
setting = re.match(r"^([\w.$()-]+)\s*(?:=|\s)\s*(.+)$", stripped)
if setting is None:
raise LauncherPatternMissing(f"{preset}:{line_no}", f"placeholder in {stripped}")
named = [p for p in substitutions if p in stripped]
if PLACEHOLDER not in stripped and not named:
# A roms placeholder the script leaves as written.
continue
key, value = setting.group(1), _unquote(setting.group(2))
for placeholder, variable in substitutions.items():
for placeholder, (variable, _line) in substitutions.items():
value = value.replace(placeholder, variable)
where = f"{section} {key}" if section else key
found += 1
applied = [(script.path, copied[0]), (script.path, number)]
found += PLACEHOLDER in stripped
applied = [(script.path, copied[0]), (script.path, number)] + [
(script.path, substitutions[p][1]) for p in named
]
self.record(
"passes_path", [], [(preset, line_no)] + applied,
search_dir=value, passes_to=f"{self.config_name(target)} {where}",
@@ -324,12 +340,13 @@ class Component:
raise LauncherPatternMissing(preset, f"no {PLACEHOLDER} line")
@staticmethod
def _substitutions(script: Script, target: str) -> dict[str, str]:
def _substitutions(script: Script, target: str) -> dict[str, tuple[str, int]]:
"""Each placeholder sed rewrites in ``target``: (variable, line)."""
found = {}
for line in script.lines:
for line_no, line in enumerate(script.lines, 1):
for m in re.finditer(r"s#(RETRODECK\w+)#\$\{?(\w+)\}?#g[\"']?\s+\"([^\"]+)\"", line):
if m.group(3) == target:
found[m.group(1)] = f"${m.group(2)}"
found[m.group(1)] = (f"${m.group(2)}", line_no)
return found
def romlists(
+4 -1
View File
@@ -809,7 +809,10 @@ def find_undeclared_files(
# Read below the roms directory (ROCKNIX's roms/pico-8)
# or in an emulator folder (RetroBat's emulators/pico8).
entry["root"] = f["root"]
elif launcher_layer and not save_root:
if launcher_layer and not save_root and f.get("root") not in OUTER_ROOTS:
# One read outside the BIOS tree is still served where a
# launcher links that very file: RetroDECK links Cemu's
# keys.txt to bios/cemu/keys.txt.
launcher_candidates.append((emu_name, is_standalone, f, entry))
keys = set_keys((emu_name, is_standalone), f)
if duplicate: