feat: read ES-DE MAME search paths

ES-DE starts standalone MAME with -rompath %GAMEDIR%\;%ROMPATH%/<system>
in 78 live commands across 54 systems (resources/systems/linux/
es_systems.xml at retrodeck-main-20261103-162710, 15e1b236; two more
are commented out). The command line outranks mame.ini, so RetroDECK's
MAME reads its sets in the roms folder each command names and in no
BIOS folder. The ES-DE rom folders parser now records, per folder, the
folders a standalone command passes as its ROM search path and the
machines it starts, or that the game is the machine (%BASENAME%).
Rescraped at the pinned revisions: retrodeck, emudeck and
emudeck-windows gain these records and nothing else.

A command reads the sets its machines load: the machine's own set, the
BIOS sets on its romof chain and the device sets with ROMs it
instantiates, from games/arcade/mame.json (mame0289 -listxml). That
index now also lists the game and device sets of each driver source
file, so a %BASENAME% folder reads for its board: neogeo and stv by
their BIOS root, cps1-3 and model2 by source file, daphne and laserdisc
by the laserdisc player devices, gameandwatch and lcdgames by the
handheld sources, any other (arcade, mame, consolearcade) for every
machine. The published installer copy of the index leaves the sources
out.

Manifests: RetroDECK gains 1399 files, 378.2 MB, below roms/. Of the 219
sets named as read outside the BIOS folder, 58 reach 146 files (25.9 MB),
161 stay named, no command's machine reading them. 441 sets RetroDECK's
list also declares at bios/ reach 1253 files (352.3 MB), 429 each in
arcade and mame and 480 in consolearcade. No other platform changes.

scripts/common.py

@@ -2872,11 +2872,15 @@ CONTENT_SCOPE_TAG = "contentscope:"
 # [core name, roms folder, (tree, path)]: a libretro core the platform
 # launches for the games of that folder, which it reads beside.
 CONTENT_FOLDER_PREFIX = "contentdir:"
+# [emulator, (tree, path), system, machines, any machine, source_ref]: a
+# standalone emulator handed a roms folder as its ROM search path on the
+# command line of a system's launch.
+ROM_SEARCH_PREFIX = "romsearch:"
 # The frontend writes the BIOS it hands a core beside the content: the
 # folder it gives the BIOS is the content's.
 CONTENT_SYSTEM_TAG = "contentsystem:"
 CONTENT_TAG_PREFIXES = (
-    CONTENT_SCOPE_TAG, CONTENT_FOLDER_PREFIX, CONTENT_SYSTEM_TAG,
+    CONTENT_SCOPE_TAG, CONTENT_FOLDER_PREFIX, ROM_SEARCH_PREFIX, CONTENT_SYSTEM_TAG,
 )
 
 
@@ -3407,7 +3411,8 @@ def _core_key(name: str) -> str:
 
 
 def content_folder_tags(config: dict) -> set[str]:
-    """The roms folders a platform launches each libretro core for, as tags.
+    """The roms folders a platform launches each libretro core for, and the
+    ROM search paths its commands hand standalone emulators, as tags.
 
     A core reading its BIOS sets beside the game (MAME 2003's osd_get_path
     gives FILETYPE_ROM the content directory) reads them in the folder of
@@ -3455,9 +3460,88 @@ def content_folder_tags(config: dict) -> set[str]:
     tags |= {
         CONTENT_FOLDER_PREFIX + json.dumps([c, f, place(f)]) for c, f in pairs if f in folders
     }
+    for row in rows:
+        for search in row.get("searched_by") or []:
+            for read in search.get("reads") or []:
+                if read in folders:
+                    tags.add(ROM_SEARCH_PREFIX + json.dumps([
+                        str(search["emulator"]), place(read), str(row["folder"]),
+                        sorted(search.get("machines") or []), bool(search.get("any_machine")),
+                        str(search.get("source_ref", "")),
+                    ]))
     return tags
 
 
+# The machines ES-DE runs from a folder whose command starts the game itself
+# (%BASENAME%), by the folder's system: its BIOS root, the driver source
+# files of its board, or a device its machines carry. A system not listed
+# holds any machine.
+ANY_MACHINE_POPULATIONS: dict[str, tuple[str, tuple[str, ...]]] = {
+    "neogeo": ("bios", ("neogeo",)),
+    "stv": ("bios", ("stvbios",)),
+    "cps": ("source", ("capcom/cps1.cpp", "capcom/cps2.cpp", "capcom/cps3.cpp")),
+    "cps1": ("source", ("capcom/cps1.cpp",)),
+    "cps2": ("source", ("capcom/cps2.cpp",)),
+    "cps3": ("source", ("capcom/cps3.cpp",)),
+    "model2": ("source", ("sega/model2.cpp",)),
+    "daphne": ("device_source", ("devices/machine/ld",)),
+    "laserdisc": ("device_source", ("devices/machine/ld",)),
+    "gameandwatch": ("source", ("handheld/",)),
+    "lcdgames": ("source", ("handheld/",)),
+}
+
+
+@functools.lru_cache(maxsize=8)
+def _arcade_relation(profile_key: str) -> dict | None:
+    """games/arcade/<profile>.json: the BIOS and device sets each game set
+    loads, from the -listxml of the version the profile pins."""
+    path = Path(__file__).resolve().parent.parent / "games" / "arcade" / f"{profile_key}.json"
+    if not path.is_file():
+        return None
+    with path.open(encoding="utf-8") as f:
+        return json.load(f)
+
+
+def rom_search_sets(
+    profile_key: str, system: str, machines: Iterable[str], any_machine: bool,
+) -> frozenset[str]:
+    """The sets a command reads from the ROM search path it passes.
+
+    The machines it starts, the BIOS sets on their romof chain and the
+    device sets with ROMs they instantiate, as the -listxml records them.
+    A command starting the game itself reads them for each machine its
+    folder holds (ANY_MACHINE_POPULATIONS).
+    """
+    return _rom_search_sets(profile_key, system, tuple(sorted(machines)), any_machine)
+
+
+@functools.lru_cache(maxsize=256)
+def _rom_search_sets(
+    profile_key: str, system: str, machines: tuple[str, ...], any_machine: bool,
+) -> frozenset[str]:
+    relation = _arcade_relation(profile_key)
+    if relation is None:
+        return frozenset()
+    bios, devices = relation.get("bios") or {}, relation.get("devices") or {}
+    sources = relation.get("sources") or {}
+    members: set[str] = set(machines)
+    kind, keys = ANY_MACHINE_POPULATIONS.get(system, ("all", ())) if any_machine else ("", ())
+    if kind == "bios":
+        members |= {g for k in keys for g in bios.get(k) or []} | set(keys)
+    elif kind == "source":
+        members |= {g for src, sets in sources.items() if src.startswith(keys) for g in sets}
+    elif kind == "device_source":
+        carried = {d for src, sets in sources.items() if src.startswith(keys) for d in sets}
+        members |= {g for d in carried for g in devices.get(d) or []}
+
+    def used(users: list[str]) -> bool:
+        return kind == "all" or not members.isdisjoint(users)
+
+    found = {b for b, users in bios.items() if used(users)}
+    found |= {d for d, users in devices.items() if used(users)}
+    return frozenset(found | members)
+
+
 def content_layout(
     emu_name: str, view: dict, standalone: bool, standalone_cores: Iterable[str],
 ) -> dict:
@@ -3465,8 +3549,11 @@ def content_layout(
 
     A libretro entry read beside the content (``load_from: content_dir``)
     goes to the folder of each system the platform launches the core for,
-    and is named as read outside the BIOS folder where it knows none.
-    Outside a platform (no CONTENT_SCOPE_TAG) the view is left as it is.
+    and is named as read outside the BIOS folder where it knows none. A
+    standalone entry no launcher record places, under the root of the
+    layout a passed ROM search path stands for, goes to each folder a
+    command passes for a machine that reads it. Outside a platform (no
+    CONTENT_SCOPE_TAG) the view is left as it is.
     """
     tags = [t for t in standalone_cores if t.startswith(CONTENT_TAG_PREFIXES)]
     if CONTENT_SYSTEM_TAG in tags and not standalone:
@@ -3511,7 +3598,33 @@ def content_layout(
                 )
                 out.append({**plain, **where, "path": f"{folder}/{rel}", "layout_path": rel})
         return {**view, "files": out} if changed else view
-    return view
+    searches = [
+        json.loads(t[len(ROM_SEARCH_PREFIX):])
+        for t in tags if t.startswith(ROM_SEARCH_PREFIX)
+    ]
+    searches = [s for s in searches if _core_key(s[0]) in names]
+    pools = {str(v).strip("/") for v in (view.get("standalone_pools") or {}).values()}
+    if not searches or "" not in pools:
+        return view
+    for f in files:
+        layout = standalone_layout_path(f, view) if isinstance(f, dict) else None
+        if not layout or "/" in layout or f.get("unplaced") != OUTSIDE_BIOS_FOLDER:
+            out.append(f)
+            continue
+        stem = layout.rsplit(".", 1)[0]
+        reads = sorted({
+            tuple(where) for _emu, where, system, machines, any_machine, _ref in searches
+            if where and stem in rom_search_sets(emu_name, system, machines, any_machine)
+        })
+        if not reads:
+            out.append(f)
+            continue
+        changed = True
+        placed = {k: v for k, v in f.items() if k != "unplaced"}
+        out.extend(
+            _placed_entry(placed, view, (tree, f"{folder}/{layout}")) for tree, folder in reads
+        )
+    return {**view, "files": out} if changed else view
 
 
 def _content_load(entry: dict) -> str:

scripts/generate_games_index.py

@@ -543,7 +543,12 @@ def build_systems(
 def build_arcade(directory: Path) -> dict[str, dict]:
     if not directory.is_dir():
         return {}
-    return {path.stem: _load_json(path) for path in sorted(directory.glob("*.json"))}
+    # The driver sources place a search path's sets at build time
+    # (common.rom_search_sets): the installer reads none of them.
+    return {
+        path.stem: {k: v for k, v in _load_json(path).items() if k != "sources"}
+        for path in sorted(directory.glob("*.json"))
+    }
 
 
 # --- Writing ----------------------------------------------------------------

scripts/scraper/arcade_parent_importer.py

@@ -222,6 +222,13 @@ def build_index(
         for device in set(entry.devices) & device_roms:
             devices.setdefault(device, set()).add(name)
     with_bios = {g for members in bios.values() for g in members}
+    # The source file of each game and device set: what a launcher that
+    # starts whatever set a folder holds selects a folder's machines by
+    # (ES-DE runs MAME on any CPS-2 set of roms/cps2).
+    sources: dict[str, set[str]] = {}
+    for name, entry in sets.items():
+        if entry.source and (name in games or name in device_roms):
+            sources.setdefault(entry.source, set()).add(archive(name, name))
     return {
         "bios": {k: sorted(v) for k, v in sorted(bios.items())},
         "devices": {k: sorted(v) for k, v in sorted(devices.items())},
@@ -229,6 +236,7 @@ def build_index(
         "missing_parents": dict(sorted(missing.items())),
         "no_bios": sorted(set(games) - with_bios - set(missing)),
         "bios_sets": sorted(bios_names),
+        **({"sources": {k: sorted(v) for k, v in sorted(sources.items())}} if sources else {}),
         "counts": {
             "sets": len(sets),
             "games": len(games),

scripts/scraper/romfolders/__init__.py

@@ -36,7 +36,7 @@ SETUP_KEYS = (
 )
 ROW_KEYS = (
     "folder", "name", "aliases", "extensions", "files", "core", "emulator", "source_ref",
-    "board_defaults", "arcade", "note",
+    "board_defaults", "arcade", "searched_by", "note",
 )
 
 

scripts/scraper/romfolders/es.py

@@ -24,6 +24,7 @@ Formats, by the shape of the source:
 
 from __future__ import annotations
 
+import html
 import json
 import re
 import shlex
@@ -167,7 +168,7 @@ def _all_lines(text: str, pattern: str) -> dict[str, list[int]]:
     for number, line in enumerate(text.splitlines(), 1):
         match = re.search(pattern, line)
         if match:
-            out.setdefault(match.group(1).strip(), []).append(number)
+            out.setdefault(html.unescape(match.group(1)).strip(), []).append(number)
     return out
 
 
@@ -522,6 +523,64 @@ def _alternative_emulators(text: str, function: str) -> dict[str, tuple[str, int
     return out
 
 
+_ROMPATH_OPTIONS = ("-rompath", "-rp")
+
+
+def _searched_by(commands: list[tuple[str, str, int | None]], key: str, path: str) -> list[dict]:
+    """The ROM folders each standalone emulator's commands pass it, by emulator.
+
+    ES-DE starts standalone MAME with `-rompath %GAMEDIR%\\;%ROMPATH%/<system>`
+    and a machine, or %BASENAME% where the game is the machine. The command
+    line outranks mame.ini. %GAMEDIR% is the game's own folder, no folder a
+    pack fills.
+    """
+    found: dict[str, dict] = {}
+    for _label, command, line in commands:
+        emulator = _EMULATOR.search(command)
+        if emulator is None or _RA_CORE.search(command):
+            continue
+        words = command.replace("\\;", ";").split()
+        at = next((i for i, w in enumerate(words) if w in _ROMPATH_OPTIONS), None)
+        if at is None or at + 1 >= len(words):
+            continue
+        reads = [
+            folder for part in words[at + 1].split(";")
+            if (folder := _strip_prefix(part, ["%ROMPATH%/", "%ROMPATH%\\"]))
+        ]
+        if not reads:
+            continue
+        name = emulator.group(1).lower()
+        record = found.setdefault(name, {"emulator": name, "reads": [], "machines": [],
+                                         "any_machine": False, "lines": []})
+        record["reads"] += [r for r in reads if r not in record["reads"]]
+        machine = words[at + 2] if at + 2 < len(words) else ""
+        if not machine or machine.startswith("%"):
+            record["any_machine"] = True
+        elif machine not in record["machines"]:
+            record["machines"].append(machine)
+        record["lines"].append(line)
+    rows = []
+    for record in found.values():
+        row = {"emulator": record["emulator"], "reads": record["reads"]}
+        if record["machines"]:
+            row["machines"] = record["machines"]
+        if record["any_machine"]:
+            row["any_machine"] = True
+        row["source_ref"] = _ref(key, path, *record["lines"])
+        rows.append(row)
+    return rows
+
+
+def _command_lines(text: str) -> dict[str, list[int]]:
+    """Command text -> the lines it is written on, in file order."""
+    out: dict[str, list[int]] = {}
+    for number, line in enumerate(text.splitlines(), 1):
+        match = re.search(r"<command\b[^>]*>([^<]*)</command>", line)
+        if match:
+            out.setdefault(match.group(1).strip(), []).append(number)
+    return out
+
+
 @register("esde")
 def esde(spec: dict, fetcher: Fetcher) -> list[dict]:
     """Each system of ES-DE's list whose path is %ROMPATH%/<folder>.
@@ -537,13 +596,21 @@ def esde(spec: dict, fetcher: Fetcher) -> list[dict]:
         key, path = item["source"], item["path"]
         text = _lenient_xml(fetcher.text(key, path))
         lines = _first_lines(text, r"<name>([^<]+)</name>")
+        command_lines = _command_lines(text)
         for name, rom_path, extensions, commands in _esde_systems(text, path):
             folder = _strip_prefix(rom_path, ["%ROMPATH%/", "%ROMPATH%\\"])
             if not name or not folder:
                 continue
+            start = lines.get(name) or 0
+            placed = [
+                (label, command, next(
+                    (n for n in command_lines.get(command, []) if n >= start), None))
+                for label, command in commands
+            ]
             systems[name] = {
                 "folder": folder, "extensions": extensions, "commands": commands,
                 "ref": _ref(key, path, lines.get(name)),
+                "searched_by": _searched_by(placed, key, path),
             }
     alternatives: dict[str, tuple[str, int]] = {}
     alt = params.get("alternatives")
@@ -580,6 +647,7 @@ def esde(spec: dict, fetcher: Fetcher) -> list[dict]:
             "emulator": emulator,
             "source_ref": "; ".join(refs),
             "note": note,
+            "searched_by": system["searched_by"] or None,
         })
     return rows