chore: merge integ configs root and switch folders

The configs root place joins roms and emulator folders in the split
handed directories, and RetroBat's portable folder table keeps both
sides. Batocera, KNULLI and RetroBat records regenerated.

scripts/scraper/batocera_launchers.py

@@ -97,6 +97,32 @@ def path_constants(source: Source) -> dict[str, str]:
     return constants
 
 
+def configs_root(cg: Configgen) -> dict[str, str]:
+    """Where configgen keeps the emulators' configs, relative to the BIOS folder.
+
+    batoceraPaths' CONFIGS (/userdata/system/configs), the folder a generator
+    starts an emulator from (shadPS4's configs/shadps4), with the lines that
+    build it and BIOS.
+    """
+    import posixpath  # noqa: PLC0415
+
+    src = cg.source(f"{cg.generators.rsplit('/', 1)[0]}/batoceraPaths.py")
+    if "CONFIGS" not in cg.paths:
+        src.fail("no CONFIGS path constant")
+    wanted: set[str] = {"CONFIGS", "BIOS"}
+    spans = []
+    for node in reversed(src.tree.body):
+        name = _assigned_name(node)
+        if name in wanted:
+            spans.append((node.lineno, node.end_lineno or node.lineno))
+            wanted |= {n.id for n in ast.walk(node.value) if isinstance(n, ast.Name)}  # type: ignore[union-attr]
+    return {
+        "path": cg.paths["CONFIGS"],
+        "from_bios": posixpath.relpath(cg.paths["CONFIGS"], cg.bios),
+        "source_ref": src.ref(spans),
+    }
+
+
 def _path_value(node: ast.AST | None, constants: dict[str, str]) -> str | None:
     if isinstance(node, ast.Name):
         return constants.get(node.id)

scripts/scraper/batocera_scraper.py

@@ -14,6 +14,7 @@ import re
 import sys
 import urllib.error
 import urllib.request
+from collections.abc import Callable, Iterable
 from pathlib import Path
 
 
@@ -28,7 +29,7 @@ from .base_scraper import (
     md5_field,
     requirement_entry,
 )
-from .batocera_launchers import extract_launchers
+from .batocera_launchers import Configgen, configs_root, extract_launchers
 
 PLATFORM_NAME = "batocera"
 
@@ -168,7 +169,8 @@ def es_systems_libretro_systems(text: str, dropped: set[str]) -> dict[str, list[
 
 
 def es_systems_by_system(
-    es_systems_text: str, defaults_text: str, dropped: set[str]
+    es_systems_text: str, defaults_text: str, dropped: set[str],
+    built: Callable[[str], bool] | None = None,
 ) -> dict[str, list[str]]:
     """system -> the cores es_systems.yml offers it, in the file's order.
 
@@ -195,6 +197,10 @@ def es_systems_by_system(
         default = defaults.get(system)
         if isinstance(default, dict):
             emulator, core = default.get("emulator"), default.get("core")
+            if (built is not None and emulator and emulator != "libretro"
+                    and str(emulator) not in names and not built(str(emulator))):
+                # A default the build lacks (configgen_cores).
+                emulator = core = None
             if emulator and emulator != "libretro":
                 names.append(str(emulator))
             if core:
@@ -472,7 +478,8 @@ def platform_systems(
 
 
 def configgen_cores(
-    defaults_text: str, es_systems_text: str, dropped: set[str]
+    defaults_text: str, es_systems_text: str, dropped: set[str],
+    built: Callable[[str], bool] | None = None,
 ) -> tuple[list[str], list[str], list[str]]:
     """Core names, standalone cores and dual-mode cores of a configgen build.
 
@@ -481,9 +488,16 @@ def configgen_cores(
     dual-mode ones those es_systems.yml also offers under libretro (dolphin
     for GameCube). `dropped` names what es_systems.yml files under an
     emulator without being one.
+
+    A default es_systems.yml does not offer names an emulator the build may
+    lack: KNULLI's defaults pick suyu for the Switch, and no package of its
+    tree, its batocera submodule or its buildroot builds suyu. `built` says
+    whether the build has a standalone emulator (packages_and_programs); a
+    default it answers no for is not run, nor its core.
     """
     data = yaml_load_upstream(defaults_text)
     cores, standalone = es_systems_emulators(es_systems_text)
+    offered = set(cores)
     libretro = es_systems_libretro(es_systems_text)
     if not cores:
         raise RuntimeError("es_systems.yml names no emulator")
@@ -492,6 +506,9 @@ def configgen_cores(
             continue
         emulator = cfg.get("emulator", "")
         core = cfg.get("core", "")
+        if (built is not None and emulator and emulator != "libretro"
+                and emulator not in offered and not built(emulator)):
+            continue
         if core:
             cores.add(core)
         if emulator and emulator != "libretro":
@@ -502,6 +519,57 @@ def configgen_cores(
     return sorted(cores - dropped), sorted(standalone - dropped), sorted(dual)
 
 
+BATOCERA_REPO = "batocera-linux/batocera.linux"
+
+
+def package_names(paths: Iterable[str]) -> set[str]:
+    """The buildroot packages a tree defines: each folder holding a Config.in."""
+    return {
+        path.rsplit("/", 2)[-2]
+        for path in paths
+        if path.endswith("/Config.in") and path.count("/") >= 1
+    }
+
+
+def repo_tree(repo: str, ref: str) -> list[str]:
+    """Every path of a GitHub repository at a revision."""
+    url = f"https://api.github.com/repos/{repo}/git/trees/{ref}?recursive=1"
+    req = urllib.request.Request(url, headers=github_headers())
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        found = json.loads(resp.read().decode("utf-8"))
+    if found.get("truncated"):
+        raise RuntimeError(f"{url}: tree truncated")
+    return [entry["path"] for entry in found.get("tree", [])]
+
+
+def submodule_revision(repo: str, path: str, ref: str) -> str:
+    """The commit a repository's submodule at `path` pins at a revision."""
+    url = f"https://api.github.com/repos/{repo}/contents/{path}?ref={ref}"
+    req = urllib.request.Request(url, headers=github_headers())
+    with urllib.request.urlopen(req, timeout=30) as resp:
+        found = json.loads(resp.read().decode("utf-8"))
+    if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"):
+        raise RuntimeError(f"{url}: {path} is not a submodule")
+    return str(found["sha"])
+
+
+def packages_and_programs(packages: set[str], launchers: list[dict]) -> Callable[[str], bool]:
+    """Whether a build has a standalone emulator.
+
+    It does when a package of that name exists, or when its launcher runs a
+    program the user puts in the BIOS folder (Batocera's lexaloffle runs
+    /userdata/bios/pico-8/pico8), which no package builds.
+    """
+    supplied = {
+        name.strip()
+        for record in launchers
+        if record.get("passes_to") == "exec"
+        and str(record.get("search_dir") or "").startswith("/userdata/bios")
+        for name in str(record.get("emulator", "")).split(",")
+    }
+    return lambda emulator: emulator in packages or emulator in supplied
+
+
 class Scraper(BaseScraper):
     """Scraper for batocera-systems Python dict."""
 
@@ -509,12 +577,26 @@ class Scraper(BaseScraper):
         self.tag = fetch_stable_tag()
         super().__init__(url=SOURCE_URL.format(tag=self.tag))
 
-    def _fetch_cores(self) -> tuple[list[str], list[str], list[str]]:
-        """Core names, standalone cores and dual-mode cores at the pinned tag."""
+    def _fetch_cores(
+        self, launchers: list[dict] | None = None
+    ) -> tuple[list[str], list[str], list[str]]:
+        """Core names, standalone cores and dual-mode cores at the pinned tag.
+
+        A default es_systems.yml does not offer counts when a package of
+        Batocera's tree or its buildroot builds it, or when its launcher
+        runs a program from the BIOS folder.
+        """
+        built = None
+        if launchers is not None:
+            paths = repo_tree(BATOCERA_REPO, self.tag)
+            buildroot = submodule_revision(BATOCERA_REPO, "buildroot", self.tag)
+            paths += repo_tree("batocera-linux/buildroot", buildroot)
+            built = packages_and_programs(package_names(paths), launchers)
         return configgen_cores(
             self._fetch_text(CONFIGGEN_DEFAULTS_URL.format(tag=self.tag)),
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)),
             _not_cores(),
+            built,
         )
 
     def _fetch_text(self, url: str) -> str:
@@ -548,7 +630,10 @@ class Scraper(BaseScraper):
             SYSTEM_SLUG_MAP,
         )
 
-        cores, standalone, dual = self._fetch_cores()
+        launchers = extract_launchers(
+            lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
+        )
+        cores, standalone, dual = self._fetch_cores(launchers)
         libretro_systems = es_systems_libretro_systems(
             self._fetch_text(ES_SYSTEMS_URL.format(tag=self.tag)), _not_cores()
         )
@@ -570,8 +655,9 @@ class Scraper(BaseScraper):
             result["dual_mode_cores"] = dual
         # The systems each libretro core runs for: its save folders.
         result["libretro_systems"] = libretro_systems
-        result["launchers"] = extract_launchers(
-            lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")
+        result["launchers"] = launchers
+        result["configs_root"] = configs_root(
+            Configgen(lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}"))
         )
         return result
 

scripts/scraper/knulli_launchers.py

@@ -49,7 +49,6 @@ try:
     )
     from .configgen_launchers import (
         Launcher,
-        _calls,
         LauncherShapeError,
         Source,
         _assigned_name,
@@ -76,7 +75,6 @@ except ImportError:
     )
     from configgen_launchers import (  # type: ignore[no-redef]
         Launcher,
-        _calls,
         LauncherShapeError,
         Source,
         _assigned_name,
@@ -1071,39 +1069,6 @@ EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = (
 )
 
 
-# Generators whose command sets the emulator's XDG data folder, the parent
-# of every user folder a yuzu-family emulator derives.
-DATA_HOME_GENERATORS = ("suyu.suyuGenerator",)
-
-
-def data_homes(cg: Configgen) -> list[Launcher]:
-    """Command(..., env={"XDG_DATA_HOME": <path>}) in the generators above.
-
-    The path need not lie in /userdata/bios: KNULLI's suyu keeps its data
-    in /userdata/saves/switch, which tells where its keys are not.
-    """
-    launchers = []
-    for module in DATA_HOME_GENERATORS:
-        src = cg.generator(module)
-        found = [
-            (_path_value(item, cg.paths), key)
-            for call in _calls(src.tree, attr="Command")
-            for keyword in call.keywords
-            if keyword.arg == "env" and isinstance(keyword.value, ast.Dict)
-            for key, item in zip(keyword.value.keys, keyword.value.values)
-            if isinstance(key, ast.Constant) and key.value == "XDG_DATA_HOME"
-        ]
-        if not found or found[0][0] is None:
-            src.fail("the command environment sets no XDG_DATA_HOME path")
-        value, node = found[0]
-        launchers.append(Launcher(
-            emulator=cg.emulator(module), build="standalone", kind="passes_path", names=[],
-            search_dir=value, passes_to="environment XDG_DATA_HOME",
-            source_ref=src.ref([(node.lineno, node.end_lineno)]),
-        ))
-    return launchers
-
-
 # KNULLI builds Batocera's emulator packages from this submodule, at the
 # batocera.linux commit it pins: Citron's path patch is there.
 BATOCERA_SUBMODULE = "batocera"
@@ -1123,7 +1088,6 @@ def extract_launchers(
     launchers += boot_disks(cg, read)
     for spec in GENERIC_SITES:
         launchers += bios_sites(cg, spec)
-    launchers += data_homes(cg)
     if batocera is not None:
         prefix = f"{BATOCERA_SUBMODULE}/"
         launchers += patched_paths(

scripts/scraper/knulli_scraper.py

@@ -41,11 +41,17 @@ from .batocera_scraper import (
     es_systems_by_system,
     es_systems_libretro_systems,
     extract_systems_dict,
+    package_names,
+    packages_and_programs,
     platform_systems,
+    repo_tree,
     requirements_from_systems,
+    submodule_revision,
 )
 from .es_emulators import attach_system_cores
+from .batocera_launchers import configs_root
 from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers
+from .knulli_launchers import configgen as knulli_configgen
 
 PLATFORM_NAME = "knulli"
 
@@ -123,11 +129,27 @@ class Scraper(BaseScraper):
         defaults = self._fetch_text(CONFIGGEN_DEFAULTS_TEMPLATE.format(ref=self.tag))
         es_systems = self._fetch_text(ES_SYSTEMS_TEMPLATE.format(ref=self.tag))
         dropped = _not_cores("batocera") | _not_cores(PLATFORM_NAME)
-        cores, standalone, dual = configgen_cores(defaults, es_systems, dropped)
+        revision = self.submodule_revision()
+        launchers = extract_launchers(
+            lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
+            lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
+        )
+        # The packages KNULLI builds come from its own tree, its batocera
+        # submodule and its buildroot submodule: a default no package of
+        # them builds (suyu) is not run.
+        paths = repo_tree(REPO, self.tag)
+        paths += repo_tree("batocera-linux/batocera.linux", revision)
+        paths += repo_tree(
+            "knulli-cfw/buildroot", submodule_revision(REPO, "buildroot", self.tag)
+        )
+        built = packages_and_programs(package_names(paths), launchers)
+        cores, standalone, dual = configgen_cores(defaults, es_systems, dropped, built)
         # Each system's own list (psx, es_systems.yml:640-653: libretro
         # pcsx_rearmed, swanstation, mednafen_psx, then duckstation and
         # duckstation-legacy).
-        attach_system_cores(systems, es_systems_by_system(es_systems, defaults, dropped))
+        attach_system_cores(
+            systems, es_systems_by_system(es_systems, defaults, dropped, built)
+        )
         result = {
             "platform": "KNULLI",
             "version": self.tag,
@@ -146,10 +168,9 @@ class Scraper(BaseScraper):
         # configgen hands a libretro core /userdata/saves/<system>, as
         # Batocera's does: the systems each core runs for are its save folders.
         result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped)
-        revision = self.submodule_revision()
-        result["launchers"] = extract_launchers(
-            lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path),
-            lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path),
+        result["launchers"] = launchers
+        result["configs_root"] = configs_root(
+            knulli_configgen(lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path))
         )
         return result
 

scripts/scraper/retrobat_launchers.py

@@ -32,6 +32,7 @@ SOURCE_FILES = (
     f"{GENERATORS}/DosBoxPure.Generator.cs",
     f"{GENERATORS}/DosBoxStaging.Generator.cs",
     f"{GENERATORS}/Duckstation.Generator.cs",
+    f"{GENERATORS}/Eden.Generator.cs",
     f"{GENERATORS}/Eka2l1.Generator.cs",
     f"{GENERATORS}/Fbneo.Generator.cs",
     f"{GENERATORS}/Flycast.Generator.cs",
@@ -57,6 +58,7 @@ SOURCE_FILES = (
     f"{GENERATORS}/Project64.Generator.cs",
     f"{GENERATORS}/Raine.Generator.cs",
     f"{GENERATORS}/Rpcs3.Generator.cs",
+    f"{GENERATORS}/Ryujinx.Generator.cs",
     f"{GENERATORS}/SSF.Generator.cs",
     f"{GENERATORS}/ScummVm.Generator.cs",
     f"{GENERATORS}/ShadPS4.Generator.cs",
@@ -68,6 +70,7 @@ SOURCE_FILES = (
     f"{GENERATORS}/Xroar.Generator.cs",
     f"{GENERATORS}/Yabasanshiro.Generator.cs",
     f"{GENERATORS}/Ymir.Generator.cs",
+    f"{GENERATORS}/Yuzu.Generator.cs",
     f"{GENERATORS}/mGBA.Generator.cs",
     FEATURES,
 )
@@ -3082,44 +3085,63 @@ def _per_system(src: Source, record: Record) -> list[Record]:
     return out
 
 
-# Generators that create a portable user folder beside the emulator's binary,
-# and the emulator each one starts: the folder AppConfig.GetFullPath gives.
+# Generators that start an emulator with a portable data folder beside its
+# binary: (generator, emulator key, binary, the code's name for the folder).
 PORTABLE_USERS = (
-    ("Citron.Generator.cs", "citron", "citron.exe"),
-    ("ShadPS4.Generator.cs", "shadps4", "shadPS4.exe"),
+    ("Citron.Generator.cs", "citron", "citron.exe", "PORTABLE_DIR"),
+    ("Eden.Generator.cs", "eden", "eden.exe", "PORTABLE_DIR"),
+    ("Ryujinx.Generator.cs", "ryujinx", "Ryujinx.exe", "DefaultPortableDir"),
+    ("ShadPS4.Generator.cs", "shadps4", "shadPS4.exe", "PORTABLE_DIR"),
+    ("Yuzu.Generator.cs", "yuzu", "yuzu.exe", "PORTABLE_DIR"),
 )
 _EMULATOR_FOLDER = (
     r"GetEmulatorPathAndExe\(emulator, out string path, out string exe, \"[^\"]+\"\)"
-    r"|string path = AppConfig\.GetFullPath\(emulator\);"
+    r"|string path = AppConfig\.GetFullPath\((?:emulator(?:\.Replace\(\"-\", \" \"\))?"
+    r"|\"\w+\")\);"
+)
+_PORTABLE_FOLDER = r'string (userFolder|setupPath) = Path\.Combine\(path, "(\w+)"\);'
+# Ryujinx's generator hands --root-data-dir saves/switch/ryujinx/portable
+# instead when that folder exists.
+_ALTERNATE_ROOT = (
+    r'string portablePath = Path\.Combine\(AppConfig\.GetFullPath\("saves"\), "switch", '
+    r'"ryujinx", "portable"\);'
 )
 
 
 def portable_users(sources: dict[str, Source]) -> list[Record]:
-    """The portable user folder a generator creates beside the binary.
+    """The portable data folder a generator gives an emulator beside its binary.
 
-    The emulator folder comes from AppConfig.GetFullPath(emulator), and the
-    generator creates <folder>/user before launching: the emulator then
-    keeps its data there (keys/, sys_modules/) in place of %APPDATA%.
+    The emulator folder comes from AppConfig.GetFullPath, and the generator
+    creates or configures <folder>/user (<folder>/portable for Ryujinx)
+    before launching: the emulator then keeps its data there (keys/,
+    sys_modules/, system/) in place of %APPDATA%.
     """
     records = []
-    for name, emulator, binary in PORTABLE_USERS:
+    for name, emulator, binary, handle in PORTABLE_USERS:
         src = sources[f"{GENERATORS}/{name}"]
         found = src.find(_EMULATOR_FOLDER, f"{emulator} folder")
-        user = src.find(r'string userFolder = Path\.Combine\(path, "(\w+)"\);', "user folder")
-        create = src.find(
-            r"Directory\.CreateDirectory\(userFolder\)", "user folder creation", user.end()
-        )
+        folder = src.find(_PORTABLE_FOLDER, "portable folder")
+        spans = [src.lines(found.start(), found.end()), src.lines(folder.start(), folder.end())]
+        if folder.group(1) == "userFolder":
+            create = src.find(
+                r"Directory\.CreateDirectory\(userFolder\)", "user folder creation", folder.end()
+            )
+            spans[-1] = src.lines(folder.start(), create.end())
+        depends = None
+        alternate = re.search(_ALTERNATE_ROOT, src.text)
+        if alternate:
+            spans.append(src.lines(alternate.start(), alternate.end()))
+            depends = "unless saves/switch/ryujinx/portable exists, passed with -r"
         records.append(_record(
             emulator,
             "standalone",
             "passes_path",
             [],
-            source_ref((src.path, [
-                src.lines(found.start(), found.end()), src.lines(user.start(), create.end()),
-            ])),
-            search_dir=f'<AppConfig.GetFullPath("{emulator}")>/{user.group(1)}',
-            passes_to=f"PORTABLE_DIR, the user folder beside {binary}",
+            source_ref((src.path, spans)),
+            search_dir=f'<AppConfig.GetFullPath("{emulator}")>/{folder.group(2)}',
+            passes_to=f"{handle}, the portable folder beside {binary}",
             fails_when_missing=False,
+            depends_on_option=depends,
         ))
     return records