chore: merge round 4 fixes (links)

Both branches extracted RetroBat's Flycast BiosPath: value_folders
already produces the record, so the dedicated extractor is dropped and
its test reads the generic one.

scripts/common.py

@@ -6,7 +6,7 @@ and file resolution - eliminates DRY violations across scripts.
 
 from __future__ import annotations
 
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from collections import Counter
 
 import hashlib
@@ -2249,6 +2249,9 @@ def runs_standalone(
 # Marks, in a platform's standalone set, an emulator it also launches as a
 # libretro core. No core name starts with it.
 DUAL_MODE_PREFIX = "dual:"
+# Marks, in the same set, a whole directory the platform's launcher hands a
+# standalone emulator (launchers.handed_directories), as JSON.
+HANDED_PREFIX = "handed:"
 
 
 def standalone_set(config: dict) -> set[str]:
@@ -2260,9 +2263,60 @@ def standalone_set(config: dict) -> set[str]:
     """
     names = {str(c) for c in config.get("standalone_cores") or []}
     names |= {DUAL_MODE_PREFIX + str(c) for c in config.get("dual_mode_cores") or []}
+    if names:
+        from launchers import handed_directories  # noqa: PLC0415
+
+        names |= {
+            HANDED_PREFIX + json.dumps(
+                [h.emulator, h.folder, h.directory, sorted(h.handles), h.source_ref]
+            )
+            for h in handed_directories(config)
+        }
     return names
 
 
+def _handed(standalone_cores: Iterable[str]) -> list:
+    """The directories a standalone set carries, as launchers built them."""
+    from launchers import HandedDirectory  # noqa: PLC0415
+
+    found = []
+    for tag in standalone_cores:
+        if not tag.startswith(HANDED_PREFIX):
+            continue
+        emulator, folder, directory, handles, ref = json.loads(tag[len(HANDED_PREFIX):])
+        found.append(HandedDirectory(emulator, folder, directory, frozenset(handles), ref))
+    return found
+
+
+def standalone_destination(entry: dict, root: str | None) -> str:
+    """Where a standalone build reads an entry, relative to the BIOS root.
+
+    ``standalone_path`` is relative to the directories the build resolves
+    it against (``standalone_directories``). A platform that hands one of
+    them a folder of its BIOS tree puts the file under that folder:
+    RetroPie and ROCKNIX give Flycast bios/dc, EmuDeck bios/flycast/bios.
+    An entry with no ``standalone_path`` keeps ``path``, the layout both
+    builds share.
+    """
+    own = entry.get("standalone_path")
+    if own and root:
+        return f"{root}/{own}"
+    return own or entry.get("path") or entry.get("name", "")
+
+
+def standalone_root_for(
+    emu_name: str, profile: dict, standalone_cores: Iterable[str]
+) -> str | None:
+    """The folder a platform gives this emulator's standalone layout, or None."""
+    declared = profile.get("standalone_directories") or []
+    if not declared:
+        return None
+    from launchers import standalone_root  # noqa: PLC0415
+
+    names = {emu_name, *(str(c) for c in profile.get("cores") or [])}
+    return standalone_root(names, declared, _handed(standalone_cores))
+
+
 def platform_layouts(
     emu_name: str, profile: dict, standalone_cores: set[str]
 ) -> list[tuple[dict, bool]]:
@@ -2275,13 +2329,27 @@ def platform_layouts(
     """
     if not runs_standalone(emu_name, profile, standalone_cores):
         return [(profile, False)]
+    standalone = profile
+    root = standalone_root_for(emu_name, profile, standalone_cores)
+    if root:
+        # The standalone view carries the platform's destination, so every
+        # reader of standalone_path (builder, manifest, verify, slots) places
+        # the file in the folder the platform hands the emulator.
+        standalone = {
+            **profile,
+            "files": [
+                {**f, "standalone_path": standalone_destination(f, root)}
+                if isinstance(f, dict) and f.get("standalone_path") else f
+                for f in profile.get("files") or []
+            ],
+        }
     dual = {c.removeprefix(DUAL_MODE_PREFIX) for c in standalone_cores
             if c.startswith(DUAL_MODE_PREFIX)}
     if "libretro" in str(profile.get("type", "")) and dual & {
         emu_name, *(str(c) for c in profile.get("cores", []))
     }:
-        return [(profile, False), (profile, True)]
-    return [(profile, True)]
+        return [(profile, False), (standalone, True)]
+    return [(standalone, True)]
 
 
 # What identifies a profile entry's content, carried unchanged wherever the

scripts/launchers.py

@@ -202,6 +202,134 @@ def launcher_places(config: dict) -> list[LauncherPlace]:
     return places
 
 
+# The names a platform's launch code gives its BIOS root, besides the
+# base_destination _bios_relative reads: RetroPie's scriptmodules set
+# biosdir="$home/RetroPie/BIOS", EmuDeck's scripts ${biosPath}, RetroDECK's
+# component scripts $bios_path.
+BIOS_ROOT_VARIABLES = ("$biosdir", "$home/RetroPie/BIOS", "${biosPath}", "$bios_path")
+
+# The kinds that hand an emulator a whole directory.
+HANDED_KINDS = ("links", "passes_path")
+
+_HOME = re.compile(r"^(?:\$\{HOME\}|\$HOME|\$home|~)(?=/|$)")
+# Flatpak sets XDG_DATA_HOME to ~/.var/app/<id>/data inside the sandbox.
+_FLATPAK_DATA = re.compile(r"^~/\.var/app/[^/]+/data(?=/|$)")
+
+
+def normalize_handle(token: str) -> str:
+    """One spelling for a path, setting or function a launcher record names.
+
+    ``$home``, ``$HOME``, ``${HOME}`` and ``~`` are one directory, and a
+    Flatpak's data directory is its ``$XDG_DATA_HOME``.
+    """
+    token = token.strip().strip("\"'").rstrip(",;").rstrip("/")
+    token = token.removesuffix("()")
+    token = _HOME.sub("~", token)
+    return _FLATPAK_DATA.sub("$XDG_DATA_HOME", token)
+
+
+def _folder_in_bios(directory: str, base: str) -> str | None:
+    """A directory relative to the BIOS root, None when it lies outside."""
+    relative = _bios_relative(directory, base)
+    if relative is not None:
+        return relative
+    directory = directory.strip().rstrip("/")
+    for variable in BIOS_ROOT_VARIABLES:
+        if directory == variable:
+            return ""
+        if directory.startswith(variable + "/"):
+            rest = directory[len(variable) + 1:]
+            return None if re.search(r"[\s<>$:;]", rest) else rest
+    return None
+
+
+@dataclass(frozen=True)
+class HandedDirectory:
+    """A whole directory a platform hands a standalone emulator.
+
+    ``folder`` is the directory relative to the BIOS root, None when it lies
+    outside it. ``handles`` are the paths, settings and functions the record
+    names on the emulator's side: the link's other end, the config key, the
+    function a patch rewrites.
+    """
+
+    emulator: str
+    folder: str | None
+    directory: str
+    handles: frozenset[str]
+    source_ref: str
+
+
+def handed_directories(config: dict) -> list[HandedDirectory]:
+    """Every whole directory a platform links or passes to a standalone build.
+
+    A record naming files (gpsp's gba_bios.bin) or one slot of a directory
+    (RetroDECK's GC/EUR) hands over no whole directory, and a search_dir
+    holding several directories names no single one.
+    """
+    base = str(config.get("base_destination") or "bios")
+    found: list[HandedDirectory] = []
+    for record in config.get("launchers") or []:
+        if record.get("build") != "standalone" or record.get("kind") not in HANDED_KINDS:
+            continue
+        if record.get("names") or record.get("slot"):
+            continue
+        raw = str(record.get("search_dir") or "").strip()
+        if not raw or re.search(r"[\s,:;]", raw):
+            continue
+        directory = normalize_handle(raw)
+        handles = frozenset(
+            handle
+            for token in str(record.get("passes_to") or "").split()
+            if (handle := normalize_handle(token)) and handle != directory
+        )
+        for emulator in (e.strip() for e in str(record.get("emulator", "")).split(",")):
+            found.append(HandedDirectory(
+                emulator=emulator,
+                folder=_folder_in_bios(raw, base),
+                directory=directory,
+                handles=handles,
+                source_ref=str(record.get("source_ref", "")),
+            ))
+    return found
+
+
+def standalone_root(
+    names: Iterable[str], declared: Iterable[str], handed: Iterable[HandedDirectory]
+) -> str | None:
+    """The folder of the BIOS tree a platform gives a standalone build's layout.
+
+    ``declared`` are the directories the build resolves ``standalone_path``
+    against, as its profile names them (``standalone_directories``). A
+    record handing a folder of the BIOS tree under one of them gives it
+    directly: RetroPie points Flycast's ~/.local/share/flycast at bios/dc. A
+    record handing a directory outside the tree under one of them is
+    followed through a link to the tree: ROCKNIX's patched Flycast uses
+    /storage/roms/dreamcast/data, which its start script links to bios/dc.
+
+    None when nothing gives a folder, or when two records give two: the
+    layout is then the profile's, relative to the BIOS root.
+    """
+    names = set(names)
+    wanted = {normalize_handle(d) for d in declared}
+    if not wanted:
+        return None
+    records = [h for h in handed if h.emulator in names]
+    folders: set[str] = set()
+    for record in records:
+        if not record.handles & wanted:
+            continue
+        if record.folder is not None:
+            folders.add(record.folder)
+            continue
+        folders.update(
+            link.folder
+            for link in records
+            if link.folder is not None and record.directory in link.handles
+        )
+    return next(iter(folders)) if len(folders) == 1 else None
+
+
 def declared_destinations(config: dict) -> set[str]:
     """Each destination the platform declares, relative to the core file directory.
 

scripts/scraper/batocera_launchers.py

@@ -647,6 +647,46 @@ def gsplus(cg: Configgen) -> list[Launcher]:
     )]
 
 
+# Generators that hand a directory of /userdata/bios to the emulator through
+# its environment, with the module holding their path constants.
+ENVIRONMENT_GENERATORS = (("flycast.flycastGenerator", "flycast/flycastPaths.py"),)
+
+
+def environment_directories(cg: Configgen) -> list[Launcher]:
+    """Command(..., env={KEY: <path under BIOS>}) in the generator."""
+    launchers = []
+    for module, paths_module in ENVIRONMENT_GENERATORS:
+        src = cg.generator(module)
+        constants = dict(cg.paths)
+        for node in cg.source(f"{GENERATORS}/{paths_module}").tree.body:
+            name = _assigned_name(node)
+            value = _path_value(getattr(node, "value", None), constants)
+            if name is not None and value is not None:
+                constants[name] = value
+        prefix = cg.bios.rstrip("/")
+        found = [
+            (key.value, value, key)
+            for call in _calls(src.tree, attr="Command")
+            for keyword in call.keywords
+            if keyword.arg == "env" and isinstance(keyword.value, ast.Dict)
+            for key, item in zip(keyword.value.keys, keyword.value.values)
+            if isinstance(key, ast.Constant)
+            and (value := _path_value(item, constants)) is not None
+            and (value == prefix or value.startswith(prefix + "/"))
+        ]
+        if not found:
+            src.fail("the command environment names no /userdata/bios directory")
+        launchers.extend(
+            Launcher(
+                emulator=cg.emulator(module), build="standalone", kind="passes_path",
+                names=[], search_dir=value, passes_to=f"environment {key}",
+                source_ref=src.ref([(node.lineno, node.end_lineno)]),
+            )
+            for key, value, node in found
+        )
+    return launchers
+
+
 # libretro core options whose value names the BIOS file the core opens.
 FILE_OPTIONS = (("_o2em_options", "o2em_bios"), ("_neocd_options", "neocd_bios"))
 
@@ -701,7 +741,7 @@ def core_file_options(cg: Configgen) -> list[Launcher]:
 
 EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = (
     duckstation, duckstation_legacy, hatari, xemu, melonds, mupen64plus,
-    x16emu, gsplus, core_file_options,
+    x16emu, gsplus, environment_directories, core_file_options,
 )
 
 

scripts/scraper/emudeck_scraper.py

@@ -231,6 +231,25 @@ def launcher_revision(url: str) -> str:
     return found.group(1)
 
 
+RETROARCH_SCRIPT = "functions/EmuScripts/emuDeckRetroArch.sh"
+
+
+def installed_libretro_cores(script: str) -> list[str]:
+    """The cores RetroArch_installCores downloads, without the _libretro suffix."""
+    body = re.search(r"RetroArch_installCores\(\)\s*\{(.*?)\n\}", script, re.S)
+    array = re.search(r"RAcores=\((.*?)\)", body.group(1), re.S) if body else None
+    if array is None:
+        raise RuntimeError(f"{RETROARCH_SCRIPT}: no RAcores list in RetroArch_installCores")
+    cores = [
+        word.removesuffix("_libretro")
+        for word in array.group(1).replace("\\", " ").split()
+        if word.endswith("_libretro")
+    ]
+    if not cores:
+        raise RuntimeError(f"{RETROARCH_SCRIPT}: RAcores names no core")
+    return cores
+
+
 class Scraper(BaseScraper):
     """Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets."""
 
@@ -451,6 +470,9 @@ class Scraper(BaseScraper):
         version = fetch_github_latest_version("dragoonDorise/EmuDeck")
 
         cores = self._fetch_installed_emulators()
+        standalone = [core for core in cores if core != "retroarch"]
+        tree = RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url))
+        libretro = set(installed_libretro_cores(tree.read(RETROARCH_SCRIPT)))
 
         return {
             "platform": "EmuDeck",
@@ -469,12 +491,14 @@ class Scraper(BaseScraper):
             # Dolphin, Cemu, xemu, ARMSX2 and Supermodel ran with the libretro
             # layout and lost their standalone-only files. A profile with no
             # standalone build keeps its layout (runs_standalone).
-            "standalone_cores": [core for core in cores if core != "retroarch"],
+            "standalone_cores": standalone,
+            # RetroArch_installCores downloads a libretro build of some of
+            # them (flycast_libretro reads system/dc/ beside the standalone
+            # Flycast's flycast/bios/): both layouts are packed.
+            "dual_mode_cores": [core for core in standalone if core in libretro],
             "systems": systems,
             # Read at the commit checkBIOS.sh comes from.
-            "launchers": extract_launchers(
-                RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url))
-            ),
+            "launchers": extract_launchers(tree),
         }
 
     def _fetch_installed_emulators(self) -> list[str]:

scripts/scraper/rocknix_launchers.py

@@ -612,6 +612,52 @@ def links(tree: RepoTree) -> list[Launcher]:
     return records
 
 
+# Packages whose build patch replaces the function an emulator resolves a
+# directory with by a fixed path: ROCKNIX's Flycast returns
+# /storage/roms/dreamcast/data as its data directory, which start_flycast.sh
+# then links to the BIOS tree.
+DIRECTORY_PATCHES = (
+    ("flycast", f"{STANDALONE}/flycast-sa/patches/000-conf-dir.patch", "find_user_data_dir"),
+)
+
+
+def patched_directories(tree: RepoTree) -> list[Launcher]:
+    records = []
+    for emulator, path, function in DIRECTORY_PATCHES:
+        lines = tree.read(path).splitlines()
+        target = next(
+            (line[len("+++ b/"):] for line in lines if line.startswith("+++ b/")), None
+        )
+        start = next(
+            (n for n, line in enumerate(lines, 1) if re.search(rf"\b{function}\(\)", line)),
+            None,
+        )
+        returned = None
+        if target is not None and start is not None:
+            returned = next(
+                (
+                    (n, match.group(1))
+                    for n, line in enumerate(lines[start:], start + 1)
+                    if (match := re.match(r'^\+\s*return "(/[^"]+)";', line))
+                ),
+                None,
+            )
+        if returned is None:
+            raise LauncherPatternMissing(path, f"{function}() patched to return a fixed path")
+        number, directory = returned
+        records.append(Launcher(
+            emulator=emulator,
+            build="standalone",
+            kind="passes_path",
+            names=[],
+            source_ref=source_ref([(path, {start, number})]),
+            search_dir=directory.rstrip("/"),
+            passes_to=f"{function}() in {target} returns it",
+            fails_when_missing=False,
+        ))
+    return records
+
+
 def _existence_guard(script: Script, link: Link) -> int | None:
     """The line of a -e test of the BIOS-side path the link waits for, if any."""
     last = max(link.lines)
@@ -639,6 +685,7 @@ EXTRACTORS: tuple[Callable[[RepoTree], list[Launcher]], ...] = (
     duckstation,
     aethersx2,
     armsx2,
+    patched_directories,
     links,
 )