chore: merge integ configs root and switch folders

The configs root place joins roms and emulator folders in the split
handed directories, and RetroBat's portable folder table keeps both
sides. Batocera, KNULLI and RetroBat records regenerated.
This commit is contained in:
Abdessamad Derraz committed 2026-11-09 20:16:41 +01:00
commit e93ff86bf1
27 files changed
+893 -206

No files matched your search

+3
View File
@@ -44,6 +44,7 @@ MODULE = "retrobios"
# keeps the exit codes it gives a reader that left (141) and Ctrl+C (130).
LAUNCHER = '''"""The RetroBIOS installer, install.py, run as the retrobios command."""
import os
import runpy
from pathlib import Path
@@ -51,6 +52,8 @@ __version__ = "{version}"
def main() -> None:
# The hints give back the command that ran this, not python3 install.py.
os.environ["RETROBIOS_BOOTSTRAP"] = "pypi"
runpy.run_path(str(Path(__file__).with_name("install.py")), run_name="__main__")
'''
+11 -1
View File
@@ -2882,7 +2882,8 @@ OUTSIDE_BIOS_FOLDER = "outside_bios_folder"
# describes. Each is located by the registry entry <root>_directory.
ROMS_ROOT = "roms"
EMULATORS_ROOT = "emulators"
OUTER_ROOTS = (ROMS_ROOT, EMULATORS_ROOT)
CONFIGS_ROOT = "configs"
OUTER_ROOTS = (ROMS_ROOT, EMULATORS_ROOT, CONFIGS_ROOT)
def outer_layout(config: dict, root: str) -> dict | None:
@@ -2915,6 +2916,15 @@ def outer_layout(config: dict, root: str) -> dict | None:
return {"path": str(path), "install_prefix": "" if inside == "." else inside}
if root == EMULATORS_ROOT:
return config.get("emulators_directory") or None
if root == CONFIGS_ROOT:
# The frontend's configs folder, as its path code builds it
# (configs_root, Batocera's batoceraPaths CONFIGS).
found = config.get("configs_root") or {}
base = str(config.get("base_destination") or "")
if not (found.get("path") and found.get("from_bios") and base):
return None
return {"path": str(found["path"]), "from_bios": str(found["from_bios"]),
"bios_folder": base}
return None
+29 -2
View File
@@ -331,6 +331,8 @@ BIOS_TREE = ""
ROMS_TREE = "roms"
# The emulator folders of a platform whose registry describes them.
EMULATORS_TREE = "emulators"
# The frontend's configs folder of a platform whose file records it.
CONFIGS_TREE = "configs"
# The tree a platform's list installs into when it starts above the BIOS
# folder: a plain dest, with no prefix.
INSTALL_TREE = "install"
@@ -494,6 +496,7 @@ def handed_directories(config: dict) -> list[HandedDirectory]:
source_ref=str(record.get("source_ref", "")),
roms_place=None if folder is not None else (
_roms_place(config, folder_in_roms(part, config))
or _configs_place(folder_in_configs(part, config))
or _emulators_place(folder_in_emulators(part, config))
),
systems=systems,
@@ -504,6 +507,11 @@ def handed_directories(config: dict) -> list[HandedDirectory]:
return found
def _configs_place(path: str | None) -> Place | None:
"""A folder below the configs root, as a place of the configs tree."""
return None if path is None else (CONFIGS_TREE, path)
def _emulators_place(path: str | None) -> Place | None:
"""A folder an emulator key names, as a place of the emulators tree."""
return None if path is None else (EMULATORS_TREE, path)
@@ -523,6 +531,18 @@ def _roms_place(config: dict, path: str | None) -> Place | None:
return (ROMS_TREE, path)
def folder_in_configs(directory: str, config: dict) -> str | None:
"""A directory relative to the frontend's configs folder, None outside it.
Only a platform whose file records its configs root has one
(``configs_root.path``): Batocera's /userdata/system/configs, where its
generators start shadPS4 from configs/shadps4.
"""
from common import CONFIGS_ROOT # noqa: PLC0415
return folder_in_root(directory, config, CONFIGS_ROOT)
def folder_in_roms(directory: str, config: dict) -> str | None:
"""A directory relative to the platform's roms directory, None outside it.
@@ -530,9 +550,16 @@ def folder_in_roms(directory: str, config: dict) -> str | None:
(``roms_root.path``): ROCKNIX's /storage/roms, which holds its BIOS
folder and the folders its launchers run programs from.
"""
from common import ROMS_ROOT, outer_layout # noqa: PLC0415
from common import ROMS_ROOT # noqa: PLC0415
roms = str((outer_layout(config, ROMS_ROOT) or {}).get("path") or "").rstrip("/")
return folder_in_root(directory, config, ROMS_ROOT)
def folder_in_root(directory: str, config: dict, root: str) -> str | None:
"""A directory relative to an outer root's path, None outside it."""
from common import outer_layout # noqa: PLC0415
roms = str((outer_layout(config, root) or {}).get("path") or "").rstrip("/")
directory = directory.strip().rstrip("/")
if not roms or not directory.startswith(roms + "/"):
return None
+26
View File
@@ -97,6 +97,32 @@ def path_constants(source: Source) -> dict[str, str]:
return constants
def configs_root(cg: Configgen) -> dict[str, str]:
"""Where configgen keeps the emulators' configs, relative to the BIOS folder.
batoceraPaths' CONFIGS (/userdata/system/configs), the folder a generator
starts an emulator from (shadPS4's configs/shadps4), with the lines that
build it and BIOS.
"""
import posixpath # noqa: PLC0415
src = cg.source(f"{cg.generators.rsplit('/', 1)[0]}/batoceraPaths.py")
if "CONFIGS" not in cg.paths:
src.fail("no CONFIGS path constant")
wanted: set[str] = {"CONFIGS", "BIOS"}
spans = []
for node in reversed(src.tree.body):
name = _assigned_name(node)
if name in wanted:
spans.append((node.lineno, node.end_lineno or node.lineno))
wanted |= {n.id for n in ast.walk(node.value) if isinstance(n, ast.Name)} # type: ignore[union-attr]
return {
"path": cg.paths["CONFIGS"],
"from_bios": posixpath.relpath(cg.paths["CONFIGS"], cg.bios),
"source_ref": src.ref(spans),
}
def _path_value(node: ast.AST | None, constants: dict[str, str]) -> str | None:
if isinstance(node, ast.Name):
return constants.get(node.id)
+94 -8
View File
@@ -14,6 +14,7 @@ import re
import sys
import urllib.error
import urllib.request
from collections.abc import Callable, Iterable
from pathlib import Path
@@ -28,7 +29,7 @@ from .base_scraper import (
md5_field,
requirement_entry,
)
from .batocera_launchers import extract_launchers
from .batocera_launchers import Configgen, configs_root, extract_launchers
PLATFORM_NAME = "batocera"
@@ -168,7 +169,8 @@ def es_systems_libretro_systems(text: str, dropped: set[str]) -> dict[str, list[
def es_systems_by_system(
es_systems_text: str, defaults_text: str, dropped: set[str]
es_systems_text: str, defaults_text: str, dropped: set[str],
built: Callable[[str], bool] | None = None,
) -> dict[str, list[str]]:
"""system -> the cores es_systems.yml offers it, in the file's order.
@@ -195,6 +197,10 @@ def es_systems_by_system(
default = defaults.get(system)
if isinstance(default, dict):
emulator, core = default.get("emulator"), default.get("core")
if (built is not None and emulator and emulator != "libretro"
and str(emulator) not in names and not built(str(emulator))):
# A default the build lacks (configgen_cores).
emulator = core = None
if emulator and emulator != "libretro":
names.append(str(emulator))
if core:
@@ -472,7 +478,8 @@ def platform_systems(
def configgen_cores(
defaults_text: str, es_systems_text: str, dropped: set[str]
defaults_text: str, es_systems_text: str, dropped: set[str],
built: Callable[[str], bool] | None = None,
) -> tuple[list[str], list[str], list[str]]:
"""Core names, standalone cores and dual-mode cores of a configgen build.
@@ -481,9 +488,16 @@ def configgen_cores(
dual-mode ones those es_systems.yml also offers under libretro (dolphin
for GameCube). `dropped` names what es_systems.yml files under an
emulator without being one.
A default es_systems.yml does not offer names an emulator the build may
lack: KNULLI's defaults pick suyu for the Switch, and no package of its
tree, its batocera submodule or its buildroot builds suyu. `built` says
whether the build has a standalone emulator (packages_and_programs); a
default it answers no for is not run, nor its core.
"""
data = yaml_load_upstream(defaults_text)
cores, standalone = es_systems_emulators(es_systems_text)
offered = set(cores)
libretro = es_systems_libretro(es_systems_text)
if not cores:
raise RuntimeError("es_systems.yml names no emulator")
@@ -492,6 +506,9 @@ def configgen_cores(
continue
emulator = cfg.get("emulator", "")
core = cfg.get("core", "")
if (built is not None and emulator and emulator != "libretro"
and emulator not in offered and not built(emulator)):
continue
if core:
cores.add(core)
if emulator and emulator != "libretro":
@@ -502,6 +519,57 @@ def configgen_cores(
return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual)
BATOCERA_REPO = "batocera-linux/batocera.linux"
def package_names(paths: Iterable[str]) -> set[str]:
"""The buildroot packages a tree defines: each folder holding a Config.in."""
return {
path.rsplit("/", 2)[-2]
for path in paths
if path.endswith("/Config.in") and path.count("/") >= 1
}
def repo_tree(repo: str, ref: str) -> list[str]:
"""Every path of a GitHub repository at a revision."""
url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
req = urllib.request.Request(url, headers=github_headers())
with urllib.request.urlopen(req, timeout=60) as resp:
found = json.loads(resp.read().decode("utf-8"))
if found.get("truncated"):
raise RuntimeError(f"{url}: tree truncated")
return [entry["path"] for entry in found.get("tree", [])]
def submodule_revision(repo: str, path: str, ref: str) -> str:
"""The commit a repository's submodule at `path` pins at a revision."""
url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}"
req = urllib.request.Request(url, headers=github_headers())
with urllib.request.urlopen(req, timeout=30) as resp:
found = json.loads(resp.read().decode("utf-8"))
if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"):
raise RuntimeError(f"{url}: {path} is not a submodule")
return str(found["sha"])
def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]:
"""Whether a build has a standalone emulator.
It does when a package of that name exists, or when its launcher runs a
program the user puts in the BIOS folder (Batocera's lexaloffle runs
/userdata/bios/pico-8/pico8), which no package builds.
"""
supplied = {
name.strip()
for record in launchers
if record.get("passes_to") == "exec"
and str(record.get("search_dir") or "").startswith("/userdata/bios")
for name in str(record.get("emulator", "")).split(",")
}
return lambda emulator: emulator in packages or emulator in supplied
class Scraper(BaseScraper):
"""Scraper for batocera-systems Python dict."""
@@ -509,12 +577,26 @@ class Scraper(BaseScraper):
self.tag = fetch_stable_tag()
super().__init__(url=SOURCE_URL.format(tag=self.tag))
def _fetch_cores(self) -> tuple[list[str], list[str], list[str]]:
"""Core names, standalone cores and dual-mode cores at the pinned tag."""
def _fetch_cores(
self, launchers: list[dict] | None = None
) -> tuple[list[str], list[str], list[str]]:
"""Core names, standalone cores and dual-mode cores at the pinned tag.
A default es_systems.yml does not offer counts when a package of
Batocera's tree or its buildroot builds it, or when its launcher
runs a program from the BIOS folder.
"""
built = None
if launchers is not None:
paths = repo_tree(BATOCERA_REPO, self.tag)
buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag)
paths += repo_tree("batocera-linux/buildroot", buildroot)
built = packages_and_programs(package_names(paths), launchers)
return configgen_cores(
self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)),
self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)),
_not_cores(),
built,
)
def _fetch_text(self, url: str) -> str:
@@ -548,7 +630,10 @@ class Scraper(BaseScraper):
SYSTEM_SLUG_MAP,
)
cores, standalone, dual = self._fetch_cores()
launchers = extract_launchers(
lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
)
cores, standalone, dual = self._fetch_cores(launchers)
libretro_systems = es_systems_libretro_systems(
self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
)
@@ -570,8 +655,9 @@ class Scraper(BaseScraper):
result["dual_mode_cores"] = dual
# The systems each libretro core runs for: its save folders.
result["libretro_systems"] = libretro_systems
result["launchers"] = extract_launchers(
lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
result["launchers"] = launchers
result["configs_root"] = configs_root(
Configgen(lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}"))
)
return result
-36
View File
@@ -49,7 +49,6 @@ try:
)
from .configgen_launchers import (
Launcher,
_calls,
LauncherShapeError,
Source,
_assigned_name,
@@ -76,7 +75,6 @@ except ImportError:
)
from configgen_launchers import ( # type: ignore[no-redef]
Launcher,
_calls,
LauncherShapeError,
Source,
_assigned_name,
@@ -1071,39 +1069,6 @@ EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = (
)
# Generators whose command sets the emulator's XDG data folder, the parent
# of every user folder a yuzu-family emulator derives.
DATA_HOME_GENERATORS = ("suyu.suyuGenerator",)
def data_homes(cg: Configgen) -> list[Launcher]:
"""Command(..., env={"XDG_DATA_HOME": <path>}) in the generators above.
The path need not lie in /userdata/bios: KNULLI's suyu keeps its data
in /userdata/saves/switch, which tells where its keys are not.
"""
launchers = []
for module in DATA_HOME_GENERATORS:
src = cg.generator(module)
found = [
(_path_value(item, cg.paths), key)
for call in _calls(src.tree, attr="Command")
for keyword in call.keywords
if keyword.arg == "env" and isinstance(keyword.value, ast.Dict)
for key, item in zip(keyword.value.keys, keyword.value.values)
if isinstance(key, ast.Constant) and key.value == "XDG_DATA_HOME"
]
if not found or found[0][0] is None:
src.fail("the command environment sets no XDG_DATA_HOME path")
value, node = found[0]
launchers.append(Launcher(
emulator=cg.emulator(module), build="standalone", kind="passes_path", names=[],
search_dir=value, passes_to="environment XDG_DATA_HOME",
source_ref=src.ref([(node.lineno, node.end_lineno)]),
))
return launchers
# KNULLI builds Batocera's emulator packages from this submodule, at the
# batocera.linux commit it pins: Citron's path patch is there.
BATOCERA_SUBMODULE = "batocera"
@@ -1123,7 +1088,6 @@ def extract_launchers(
launchers += boot_disks(cg, read)
for spec in GENERIC_SITES:
launchers += bios_sites(cg, spec)
launchers += data_homes(cg)
if batocera is not None:
prefix = f"{BATOCERA_SUBMODULE}/"
launchers += patched_paths(
+27 -6
View File
@@ -41,11 +41,17 @@ from .batocera_scraper import (
es_systems_by_system,
es_systems_libretro_systems,
extract_systems_dict,
package_names,
packages_and_programs,
platform_systems,
repo_tree,
requirements_from_systems,
submodule_revision,
)
from .es_emulators import attach_system_cores
from .batocera_launchers import configs_root
from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers
from .knulli_launchers import configgen as knulli_configgen
PLATFORM_NAME = "knulli"
@@ -123,11 +129,27 @@ class Scraper(BaseScraper):
defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag))
es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))
dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME)
cores, standalone, dual = configgen_cores(defaults, es_systems, dropped)
revision = self.submodule_revision()
launchers = extract_launchers(
lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
)
# The packages KNULLI builds come from its own tree, its batocera
# submodule and its buildroot submodule: a default no package of
# them builds (suyu) is not run.
paths = repo_tree(REPO, self.tag)
paths += repo_tree("batocera-linux/batocera.linux", revision)
paths += repo_tree(
"knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag)
)
built = packages_and_programs(package_names(paths), launchers)
cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built)
# Each system's own list (psx, es_systems.yml:640-653: libretro
# pcsx_rearmed, swanstation, mednafen_psx, then duckstation and
# duckstation-legacy).
attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped))
attach_system_cores(
systems, es_systems_by_system(es_systems, defaults, dropped, built)
)
result = {
"platform": "KNULLI",
"version": self.tag,
@@ -146,10 +168,9 @@ class Scraper(BaseScraper):
# configgen hands a libretro core /userdata/saves/<system>, as
# Batocera's does: the systems each core runs for are its save folders.
result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)
revision = self.submodule_revision()
result["launchers"] = extract_launchers(
lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
result["launchers"] = launchers
result["configs_root"] = configs_root(
knulli_configgen(lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path))
)
return result
+41 -19
View File
@@ -32,6 +32,7 @@ SOURCE_FILES = (
f"{GENERATORS}/DosBoxPure.Generator.cs",
f"{GENERATORS}/DosBoxStaging.Generator.cs",
f"{GENERATORS}/Duckstation.Generator.cs",
f"{GENERATORS}/Eden.Generator.cs",
f"{GENERATORS}/Eka2l1.Generator.cs",
f"{GENERATORS}/Fbneo.Generator.cs",
f"{GENERATORS}/Flycast.Generator.cs",
@@ -57,6 +58,7 @@ SOURCE_FILES = (
f"{GENERATORS}/Project64.Generator.cs",
f"{GENERATORS}/Raine.Generator.cs",
f"{GENERATORS}/Rpcs3.Generator.cs",
f"{GENERATORS}/Ryujinx.Generator.cs",
f"{GENERATORS}/SSF.Generator.cs",
f"{GENERATORS}/ScummVm.Generator.cs",
f"{GENERATORS}/ShadPS4.Generator.cs",
@@ -68,6 +70,7 @@ SOURCE_FILES = (
f"{GENERATORS}/Xroar.Generator.cs",
f"{GENERATORS}/Yabasanshiro.Generator.cs",
f"{GENERATORS}/Ymir.Generator.cs",
f"{GENERATORS}/Yuzu.Generator.cs",
f"{GENERATORS}/mGBA.Generator.cs",
FEATURES,
)
@@ -3082,44 +3085,63 @@ def _per_system(src: Source, record: Record) -> list[Record]:
return out
# Generators that create a portable user folder beside the emulator's binary,
# and the emulator each one starts: the folder AppConfig.GetFullPath gives.
# Generators that start an emulator with a portable data folder beside its
# binary: (generator, emulator key, binary, the code's name for the folder).
PORTABLE_USERS = (
("Citron.Generator.cs", "citron", "citron.exe"),
("ShadPS4.Generator.cs", "shadps4", "shadPS4.exe"),
("Citron.Generator.cs", "citron", "citron.exe", "PORTABLE_DIR"),
("Eden.Generator.cs", "eden", "eden.exe", "PORTABLE_DIR"),
("Ryujinx.Generator.cs", "ryujinx", "Ryujinx.exe", "DefaultPortableDir"),
("ShadPS4.Generator.cs", "shadps4", "shadPS4.exe", "PORTABLE_DIR"),
("Yuzu.Generator.cs", "yuzu", "yuzu.exe", "PORTABLE_DIR"),
)
_EMULATOR_FOLDER = (
r"GetEmulatorPathAndExe\(emulator, out string path, out string exe, \"[^\"]+\"\)"
r"|string path = AppConfig\.GetFullPath\(emulator\);"
r"|string path = AppConfig\.GetFullPath\((?:emulator(?:\.Replace\(\"-\", \" \"\))?"
r"|\"\w+\")\);"
)
_PORTABLE_FOLDER = r'string (userFolder|setupPath) = Path\.Combine\(path, "(\w+)"\);'
# Ryujinx's generator hands --root-data-dir saves/switch/ryujinx/portable
# instead when that folder exists.
_ALTERNATE_ROOT = (
r'string portablePath = Path\.Combine\(AppConfig\.GetFullPath\("saves"\), "switch", '
r'"ryujinx", "portable"\);'
)
def portable_users(sources: dict[str, Source]) -> list[Record]:
"""The portable user folder a generator creates beside the binary.
"""The portable data folder a generator gives an emulator beside its binary.
The emulator folder comes from AppConfig.GetFullPath(emulator), and the
generator creates <folder>/user before launching: the emulator then
keeps its data there (keys/, sys_modules/) in place of %APPDATA%.
The emulator folder comes from AppConfig.GetFullPath, and the generator
creates or configures <folder>/user (<folder>/portable for Ryujinx)
before launching: the emulator then keeps its data there (keys/,
sys_modules/, system/) in place of %APPDATA%.
"""
records = []
for name, emulator, binary in PORTABLE_USERS:
for name, emulator, binary, handle in PORTABLE_USERS:
src = sources[f"{GENERATORS}/{name}"]
found = src.find(_EMULATOR_FOLDER, f"{emulator} folder")
user = src.find(r'string userFolder = Path\.Combine\(path, "(\w+)"\);', "user folder")
create = src.find(
r"Directory\.CreateDirectory\(userFolder\)", "user folder creation", user.end()
)
folder = src.find(_PORTABLE_FOLDER, "portable folder")
spans = [src.lines(found.start(), found.end()), src.lines(folder.start(), folder.end())]
if folder.group(1) == "userFolder":
create = src.find(
r"Directory\.CreateDirectory\(userFolder\)", "user folder creation", folder.end()
)
spans[-1] = src.lines(folder.start(), create.end())
depends = None
alternate = re.search(_ALTERNATE_ROOT, src.text)
if alternate:
spans.append(src.lines(alternate.start(), alternate.end()))
depends = "unless saves/switch/ryujinx/portable exists, passed with -r"
records.append(_record(
emulator,
"standalone",
"passes_path",
[],
source_ref((src.path, [
src.lines(found.start(), found.end()), src.lines(user.start(), create.end()),
])),
search_dir=f'<AppConfig.GetFullPath("{emulator}")>/{user.group(1)}',
passes_to=f"PORTABLE_DIR, the user folder beside {binary}",
source_ref((src.path, spans)),
search_dir=f'<AppConfig.GetFullPath("{emulator}")>/{folder.group(2)}',
passes_to=f"{handle}, the portable folder beside {binary}",
fails_when_missing=False,
depends_on_option=depends,
))
return records
+1
View File
@@ -980,6 +980,7 @@ _UNPLACED_REASONS = {
"outside_bios_folder": "read in a folder the platform keeps outside the BIOS tree",
"no_roms_directory": "read below a roms folder the platform's registry does not name",
"no_emulators_directory": "read in an emulator folder the platform's registry does not describe",
"no_configs_directory": "read below a configs folder the platform's file does not record",
"no_platform_slug": "no platform slug for its system",
"below_platform_folder": "below the folder the platform lists",
"no_save_directory": "read below a save directory the platform keeps in no fixed place",