chore: merge configs root and switch folders

This commit is contained in:
Abdessamad Derraz committed 2026-11-09 20:09:05 +01:00
commit 24eab97bc7
24 files changed
+826 -203

No files matched your search

+11 -1
View File
@@ -2825,7 +2825,8 @@ OUTSIDE_BIOS_FOLDER = "outside_bios_folder"
# describes. Each is located by the registry entry <root>_directory.
ROMS_ROOT = "roms"
EMULATORS_ROOT = "emulators"
OUTER_ROOTS = (ROMS_ROOT, EMULATORS_ROOT)
CONFIGS_ROOT = "configs"
OUTER_ROOTS = (ROMS_ROOT, EMULATORS_ROOT, CONFIGS_ROOT)
def outer_layout(config: dict, root: str) -> dict | None:
@@ -2858,6 +2859,15 @@ def outer_layout(config: dict, root: str) -> dict | None:
return {"path": str(path), "install_prefix": "" if inside == "." else inside}
if root == EMULATORS_ROOT:
return config.get("emulators_directory") or None
if root == CONFIGS_ROOT:
# The frontend's configs folder, as its path code builds it
# (configs_root, Batocera's batoceraPaths CONFIGS).
found = config.get("configs_root") or {}
base = str(config.get("base_destination") or "")
if not (found.get("path") and found.get("from_bios") and base):
return None
return {"path": str(found["path"]), "from_bios": str(found["from_bios"]),
"bios_folder": base}
return None
+29 -2
View File
@@ -331,6 +331,8 @@ BIOS_TREE = ""
ROMS_TREE = "roms"
# The emulator folders of a platform whose registry describes them.
EMULATORS_TREE = "emulators"
# The frontend's configs folder of a platform whose file records it.
CONFIGS_TREE = "configs"
# The tree a platform's list installs into when it starts above the BIOS
# folder: a plain dest, with no prefix.
INSTALL_TREE = "install"
@@ -433,12 +435,18 @@ def handed_directories(config: dict) -> list[HandedDirectory]:
source_ref=str(record.get("source_ref", "")),
roms_place=None if folder is not None else (
_roms_place(config, folder_in_roms(raw, config))
or _configs_place(folder_in_configs(raw, config))
or _emulators_place(folder_in_emulators(raw, config))
),
))
return found
def _configs_place(path: str | None) -> Place | None:
"""A folder below the configs root, as a place of the configs tree."""
return None if path is None else (CONFIGS_TREE, path)
def _emulators_place(path: str | None) -> Place | None:
"""A folder an emulator key names, as a place of the emulators tree."""
return None if path is None else (EMULATORS_TREE, path)
@@ -458,6 +466,18 @@ def _roms_place(config: dict, path: str | None) -> Place | None:
return (ROMS_TREE, path)
def folder_in_configs(directory: str, config: dict) -> str | None:
"""A directory relative to the frontend's configs folder, None outside it.
Only a platform whose file records its configs root has one
(``configs_root.path``): Batocera's /userdata/system/configs, where its
generators start shadPS4 from configs/shadps4.
"""
from common import CONFIGS_ROOT # noqa: PLC0415
return folder_in_root(directory, config, CONFIGS_ROOT)
def folder_in_roms(directory: str, config: dict) -> str | None:
"""A directory relative to the platform's roms directory, None outside it.
@@ -465,9 +485,16 @@ def folder_in_roms(directory: str, config: dict) -> str | None:
(``roms_root.path``): ROCKNIX's /storage/roms, which holds its BIOS
folder and the folders its launchers run programs from.
"""
from common import ROMS_ROOT, outer_layout # noqa: PLC0415
from common import ROMS_ROOT # noqa: PLC0415
roms = str((outer_layout(config, ROMS_ROOT) or {}).get("path") or "").rstrip("/")
return folder_in_root(directory, config, ROMS_ROOT)
def folder_in_root(directory: str, config: dict, root: str) -> str | None:
"""A directory relative to an outer root's path, None outside it."""
from common import outer_layout # noqa: PLC0415
roms = str((outer_layout(config, root) or {}).get("path") or "").rstrip("/")
directory = directory.strip().rstrip("/")
if not roms or not directory.startswith(roms + "/"):
return None
+26
View File
@@ -97,6 +97,32 @@ def path_constants(source: Source) -> dict[str, str]:
return constants
def configs_root(cg: Configgen) -> dict[str, str]:
"""Where configgen keeps the emulators' configs, relative to the BIOS folder.
batoceraPaths' CONFIGS (/userdata/system/configs), the folder a generator
starts an emulator from (shadPS4's configs/shadps4), with the lines that
build it and BIOS.
"""
import posixpath # noqa: PLC0415
src = cg.source(f"{cg.generators.rsplit('/', 1)[0]}/batoceraPaths.py")
if "CONFIGS" not in cg.paths:
src.fail("no CONFIGS path constant")
wanted: set[str] = {"CONFIGS", "BIOS"}
spans = []
for node in reversed(src.tree.body):
name = _assigned_name(node)
if name in wanted:
spans.append((node.lineno, node.end_lineno or node.lineno))
wanted |= {n.id for n in ast.walk(node.value) if isinstance(n, ast.Name)} # type: ignore[union-attr]
return {
"path": cg.paths["CONFIGS"],
"from_bios": posixpath.relpath(cg.paths["CONFIGS"], cg.bios),
"source_ref": src.ref(spans),
}
def _path_value(node: ast.AST | None, constants: dict[str, str]) -> str | None:
if isinstance(node, ast.Name):
return constants.get(node.id)
+94 -8
View File
@@ -14,6 +14,7 @@ import re
import sys
import urllib.error
import urllib.request
from collections.abc import Callable, Iterable
from pathlib import Path
@@ -28,7 +29,7 @@ from .base_scraper import (
md5_field,
requirement_entry,
)
from .batocera_launchers import extract_launchers
from .batocera_launchers import Configgen, configs_root, extract_launchers
PLATFORM_NAME = "batocera"
@@ -168,7 +169,8 @@ def es_systems_libretro_systems(text: str, dropped: set[str]) -> dict[str, list[
def es_systems_by_system(
es_systems_text: str, defaults_text: str, dropped: set[str]
es_systems_text: str, defaults_text: str, dropped: set[str],
built: Callable[[str], bool] | None = None,
) -> dict[str, list[str]]:
"""system -> the cores es_systems.yml offers it, in the file's order.
@@ -195,6 +197,10 @@ def es_systems_by_system(
default = defaults.get(system)
if isinstance(default, dict):
emulator, core = default.get("emulator"), default.get("core")
if (built is not None and emulator and emulator != "libretro"
and str(emulator) not in names and not built(str(emulator))):
# A default the build lacks (configgen_cores).
emulator = core = None
if emulator and emulator != "libretro":
names.append(str(emulator))
if core:
@@ -472,7 +478,8 @@ def platform_systems(
def configgen_cores(
defaults_text: str, es_systems_text: str, dropped: set[str]
defaults_text: str, es_systems_text: str, dropped: set[str],
built: Callable[[str], bool] | None = None,
) -> tuple[list[str], list[str], list[str]]:
"""Core names, standalone cores and dual-mode cores of a configgen build.
@@ -481,9 +488,16 @@ def configgen_cores(
dual-mode ones those es_systems.yml also offers under libretro (dolphin
for GameCube). `dropped` names what es_systems.yml files under an
emulator without being one.
A default es_systems.yml does not offer names an emulator the build may
lack: KNULLI's defaults pick suyu for the Switch, and no package of its
tree, its batocera submodule or its buildroot builds suyu. `built` says
whether the build has a standalone emulator (packages_and_programs); a
default it answers no for is not run, nor its core.
"""
data = yaml_load_upstream(defaults_text)
cores, standalone = es_systems_emulators(es_systems_text)
offered = set(cores)
libretro = es_systems_libretro(es_systems_text)
if not cores:
raise RuntimeError("es_systems.yml names no emulator")
@@ -492,6 +506,9 @@ def configgen_cores(
continue
emulator = cfg.get("emulator", "")
core = cfg.get("core", "")
if (built is not None and emulator and emulator != "libretro"
and emulator not in offered and not built(emulator)):
continue
if core:
cores.add(core)
if emulator and emulator != "libretro":
@@ -502,6 +519,57 @@ def configgen_cores(
return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual)
BATOCERA_REPO = "batocera-linux/batocera.linux"
def package_names(paths: Iterable[str]) -> set[str]:
"""The buildroot packages a tree defines: each folder holding a Config.in."""
return {
path.rsplit("/", 2)[-2]
for path in paths
if path.endswith("/Config.in") and path.count("/") >= 1
}
def repo_tree(repo: str, ref: str) -> list[str]:
"""Every path of a GitHub repository at a revision."""
url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
req = urllib.request.Request(url, headers=github_headers())
with urllib.request.urlopen(req, timeout=60) as resp:
found = json.loads(resp.read().decode("utf-8"))
if found.get("truncated"):
raise RuntimeError(f"{url}: tree truncated")
return [entry["path"] for entry in found.get("tree", [])]
def submodule_revision(repo: str, path: str, ref: str) -> str:
"""The commit a repository's submodule at `path` pins at a revision."""
url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}"
req = urllib.request.Request(url, headers=github_headers())
with urllib.request.urlopen(req, timeout=30) as resp:
found = json.loads(resp.read().decode("utf-8"))
if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"):
raise RuntimeError(f"{url}: {path} is not a submodule")
return str(found["sha"])
def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]:
"""Whether a build has a standalone emulator.
It does when a package of that name exists, or when its launcher runs a
program the user puts in the BIOS folder (Batocera's lexaloffle runs
/userdata/bios/pico-8/pico8), which no package builds.
"""
supplied = {
name.strip()
for record in launchers
if record.get("passes_to") == "exec"
and str(record.get("search_dir") or "").startswith("/userdata/bios")
for name in str(record.get("emulator", "")).split(",")
}
return lambda emulator: emulator in packages or emulator in supplied
class Scraper(BaseScraper):
"""Scraper for batocera-systems Python dict."""
@@ -509,12 +577,26 @@ class Scraper(BaseScraper):
self.tag = fetch_stable_tag()
super().__init__(url=SOURCE_URL.format(tag=self.tag))
def _fetch_cores(self) -> tuple[list[str], list[str], list[str]]:
"""Core names, standalone cores and dual-mode cores at the pinned tag."""
def _fetch_cores(
self, launchers: list[dict] | None = None
) -> tuple[list[str], list[str], list[str]]:
"""Core names, standalone cores and dual-mode cores at the pinned tag.
A default es_systems.yml does not offer counts when a package of
Batocera's tree or its buildroot builds it, or when its launcher
runs a program from the BIOS folder.
"""
built = None
if launchers is not None:
paths = repo_tree(BATOCERA_REPO, self.tag)
buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag)
paths += repo_tree("batocera-linux/buildroot", buildroot)
built = packages_and_programs(package_names(paths), launchers)
return configgen_cores(
self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)),
self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)),
_not_cores(),
built,
)
def _fetch_text(self, url: str) -> str:
@@ -548,7 +630,10 @@ class Scraper(BaseScraper):
SYSTEM_SLUG_MAP,
)
cores, standalone, dual = self._fetch_cores()
launchers = extract_launchers(
lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
)
cores, standalone, dual = self._fetch_cores(launchers)
libretro_systems = es_systems_libretro_systems(
self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
)
@@ -570,8 +655,9 @@ class Scraper(BaseScraper):
result["dual_mode_cores"] = dual
# The systems each libretro core runs for: its save folders.
result["libretro_systems"] = libretro_systems
result["launchers"] = extract_launchers(
lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
result["launchers"] = launchers
result["configs_root"] = configs_root(
Configgen(lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}"))
)
return result
-36
View File
@@ -48,7 +48,6 @@ try:
)
from .configgen_launchers import (
Launcher,
_calls,
LauncherShapeError,
Source,
_assigned_name,
@@ -74,7 +73,6 @@ except ImportError:
)
from configgen_launchers import ( # type: ignore[no-redef]
Launcher,
_calls,
LauncherShapeError,
Source,
_assigned_name,
@@ -1062,39 +1060,6 @@ EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = (
)
# Generators whose command sets the emulator's XDG data folder, the parent
# of every user folder a yuzu-family emulator derives.
DATA_HOME_GENERATORS = ("suyu.suyuGenerator",)
def data_homes(cg: Configgen) -> list[Launcher]:
"""Command(..., env={"XDG_DATA_HOME": <path>}) in the generators above.
The path need not lie in /userdata/bios: KNULLI's suyu keeps its data
in /userdata/saves/switch, which tells where its keys are not.
"""
launchers = []
for module in DATA_HOME_GENERATORS:
src = cg.generator(module)
found = [
(_path_value(item, cg.paths), key)
for call in _calls(src.tree, attr="Command")
for keyword in call.keywords
if keyword.arg == "env" and isinstance(keyword.value, ast.Dict)
for key, item in zip(keyword.value.keys, keyword.value.values)
if isinstance(key, ast.Constant) and key.value == "XDG_DATA_HOME"
]
if not found or found[0][0] is None:
src.fail("the command environment sets no XDG_DATA_HOME path")
value, node = found[0]
launchers.append(Launcher(
emulator=cg.emulator(module), build="standalone", kind="passes_path", names=[],
search_dir=value, passes_to="environment XDG_DATA_HOME",
source_ref=src.ref([(node.lineno, node.end_lineno)]),
))
return launchers
# KNULLI builds Batocera's emulator packages from this submodule, at the
# batocera.linux commit it pins: Citron's path patch is there.
BATOCERA_SUBMODULE = "batocera"
@@ -1114,7 +1079,6 @@ def extract_launchers(
launchers += boot_disks(cg, read)
for spec in GENERIC_SITES:
launchers += bios_sites(cg, spec)
launchers += data_homes(cg)
if batocera is not None:
prefix = f"{BATOCERA_SUBMODULE}/"
launchers += patched_paths(
+27 -6
View File
@@ -41,11 +41,17 @@ from .batocera_scraper import (
es_systems_by_system,
es_systems_libretro_systems,
extract_systems_dict,
package_names,
packages_and_programs,
platform_systems,
repo_tree,
requirements_from_systems,
submodule_revision,
)
from .es_emulators import attach_system_cores
from .batocera_launchers import configs_root
from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers
from .knulli_launchers import configgen as knulli_configgen
PLATFORM_NAME = "knulli"
@@ -123,11 +129,27 @@ class Scraper(BaseScraper):
defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag))
es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))
dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME)
cores, standalone, dual = configgen_cores(defaults, es_systems, dropped)
revision = self.submodule_revision()
launchers = extract_launchers(
lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
)
# The packages KNULLI builds come from its own tree, its batocera
# submodule and its buildroot submodule: a default no package of
# them builds (suyu) is not run.
paths = repo_tree(REPO, self.tag)
paths += repo_tree("batocera-linux/batocera.linux", revision)
paths += repo_tree(
"knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag)
)
built = packages_and_programs(package_names(paths), launchers)
cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built)
# Each system's own list (psx, es_systems.yml:640-653: libretro
# pcsx_rearmed, swanstation, mednafen_psx, then duckstation and
# duckstation-legacy).
attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped))
attach_system_cores(
systems, es_systems_by_system(es_systems, defaults, dropped, built)
)
result = {
"platform": "KNULLI",
"version": self.tag,
@@ -146,10 +168,9 @@ class Scraper(BaseScraper):
# configgen hands a libretro core /userdata/saves/<system>, as
# Batocera's does: the systems each core runs for are its save folders.
result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)
revision = self.submodule_revision()
result["launchers"] = extract_launchers(
lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
result["launchers"] = launchers
result["configs_root"] = configs_root(
knulli_configgen(lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path))
)
return result
+41 -19
View File
@@ -32,6 +32,7 @@ SOURCE_FILES = (
f"{GENERATORS}/DosBoxPure.Generator.cs",
f"{GENERATORS}/DosBoxStaging.Generator.cs",
f"{GENERATORS}/Duckstation.Generator.cs",
f"{GENERATORS}/Eden.Generator.cs",
f"{GENERATORS}/Eka2l1.Generator.cs",
f"{GENERATORS}/Fbneo.Generator.cs",
f"{GENERATORS}/Flycast.Generator.cs",
@@ -57,6 +58,7 @@ SOURCE_FILES = (
f"{GENERATORS}/Project64.Generator.cs",
f"{GENERATORS}/Raine.Generator.cs",
f"{GENERATORS}/Rpcs3.Generator.cs",
f"{GENERATORS}/Ryujinx.Generator.cs",
f"{GENERATORS}/SSF.Generator.cs",
f"{GENERATORS}/ScummVm.Generator.cs",
f"{GENERATORS}/ShadPS4.Generator.cs",
@@ -68,6 +70,7 @@ SOURCE_FILES = (
f"{GENERATORS}/Xroar.Generator.cs",
f"{GENERATORS}/Yabasanshiro.Generator.cs",
f"{GENERATORS}/Ymir.Generator.cs",
f"{GENERATORS}/Yuzu.Generator.cs",
f"{GENERATORS}/mGBA.Generator.cs",
FEATURES,
)
@@ -2982,44 +2985,63 @@ def xroar(sources: dict[str, Source]) -> list[Record]:
return _covered(src, records)
# Generators that create a portable user folder beside the emulator's binary,
# and the emulator each one starts: the folder AppConfig.GetFullPath gives.
# Generators that start an emulator with a portable data folder beside its
# binary: (generator, emulator key, binary, the code's name for the folder).
PORTABLE_USERS = (
("Citron.Generator.cs", "citron", "citron.exe"),
("ShadPS4.Generator.cs", "shadps4", "shadPS4.exe"),
("Citron.Generator.cs", "citron", "citron.exe", "PORTABLE_DIR"),
("Eden.Generator.cs", "eden", "eden.exe", "PORTABLE_DIR"),
("Ryujinx.Generator.cs", "ryujinx", "Ryujinx.exe", "DefaultPortableDir"),
("ShadPS4.Generator.cs", "shadps4", "shadPS4.exe", "PORTABLE_DIR"),
("Yuzu.Generator.cs", "yuzu", "yuzu.exe", "PORTABLE_DIR"),
)
_EMULATOR_FOLDER = (
r"GetEmulatorPathAndExe\(emulator, out string path, out string exe, \"[^\"]+\"\)"
r"|string path = AppConfig\.GetFullPath\(emulator\);"
r"|string path = AppConfig\.GetFullPath\((?:emulator(?:\.Replace\(\"-\", \" \"\))?"
r"|\"\w+\")\);"
)
_PORTABLE_FOLDER = r'string (userFolder|setupPath) = Path\.Combine\(path, "(\w+)"\);'
# Ryujinx's generator hands --root-data-dir saves/switch/ryujinx/portable
# instead when that folder exists.
_ALTERNATE_ROOT = (
r'string portablePath = Path\.Combine\(AppConfig\.GetFullPath\("saves"\), "switch", '
r'"ryujinx", "portable"\);'
)
def portable_users(sources: dict[str, Source]) -> list[Record]:
"""The portable user folder a generator creates beside the binary.
"""The portable data folder a generator gives an emulator beside its binary.
The emulator folder comes from AppConfig.GetFullPath(emulator), and the
generator creates <folder>/user before launching: the emulator then
keeps its data there (keys/, sys_modules/) in place of %APPDATA%.
The emulator folder comes from AppConfig.GetFullPath, and the generator
creates or configures <folder>/user (<folder>/portable for Ryujinx)
before launching: the emulator then keeps its data there (keys/,
sys_modules/, system/) in place of %APPDATA%.
"""
records = []
for name, emulator, binary in PORTABLE_USERS:
for name, emulator, binary, handle in PORTABLE_USERS:
src = sources[f"{GENERATORS}/{name}"]
found = src.find(_EMULATOR_FOLDER, f"{emulator} folder")
user = src.find(r'string userFolder = Path\.Combine\(path, "(\w+)"\);', "user folder")
create = src.find(
r"Directory\.CreateDirectory\(userFolder\)", "user folder creation", user.end()
)
folder = src.find(_PORTABLE_FOLDER, "portable folder")
spans = [src.lines(found.start(), found.end()), src.lines(folder.start(), folder.end())]
if folder.group(1) == "userFolder":
create = src.find(
r"Directory\.CreateDirectory\(userFolder\)", "user folder creation", folder.end()
)
spans[-1] = src.lines(folder.start(), create.end())
depends = None
alternate = re.search(_ALTERNATE_ROOT, src.text)
if alternate:
spans.append(src.lines(alternate.start(), alternate.end()))
depends = "unless saves/switch/ryujinx/portable exists, passed with -r"
records.append(_record(
emulator,
"standalone",
"passes_path",
[],
source_ref((src.path, [
src.lines(found.start(), found.end()), src.lines(user.start(), create.end()),
])),
search_dir=f'<AppConfig.GetFullPath("{emulator}")>/{user.group(1)}',
passes_to=f"PORTABLE_DIR, the user folder beside {binary}",
source_ref((src.path, spans)),
search_dir=f'<AppConfig.GetFullPath("{emulator}")>/{folder.group(2)}',
passes_to=f"{handle}, the portable folder beside {binary}",
fails_when_missing=False,
depends_on_option=depends,
))
return records
+1
View File
@@ -980,6 +980,7 @@ _UNPLACED_REASONS = {
"outside_bios_folder": "read in a folder the platform keeps outside the BIOS tree",
"no_roms_directory": "read below a roms folder the platform's registry does not name",
"no_emulators_directory": "read in an emulator folder the platform's registry does not describe",
"no_configs_directory": "read below a configs folder the platform's file does not record",
"no_platform_slug": "no platform slug for its system",
"below_platform_folder": "below the folder the platform lists",
"no_save_directory": "read below a save directory the platform keeps in no fixed place",