chore: merge the MAME search paths

scripts/common.py

@@ -2760,27 +2760,49 @@ def expand_directory_entries(
 
     if not any(entry.get("type") == "directory" for entry in files):
         return files
+    # A file the profile names itself is left to that entry, in the same
+    # build and below the same root: MAME's libretro floppy samples share
+    # their path with the standalone tree RetroPie reads below roms/.
+    other_build = "libretro" if standalone else "standalone"
     named = {
-        destination(entry) for entry in files if entry.get("type") != "directory"
+        (entry.get("root") or "", destination(entry)) for entry in files
+        if entry.get("type") != "directory" and entry.get("mode") != other_build
     }
     expanded: list[dict] = []
     for entry in files:
         if entry.get("type") != "directory":
             expanded.append(entry)
             continue
-        members = directory_members(db, destination(entry))
+        target = destination(entry)
+        # A directory a platform moved (platform_layouts) is the profile's
+        # tree at the platform's place: the collection holds it under the
+        # layout path, and the platform's folder proves nothing (RetroDECK
+        # hands MAME samplepath $bios_path/mame-sa/samples).
+        layout = str(entry.get("layout_path") or "") if standalone else ""
+        members = directory_members(db, layout) if layout else {}
+        if not members:
+            # A layout tail two trees answer (MAME's hash/) leaves the
+            # platform's own tail.
+            layout = ""
+            members = directory_members(db, target)
+        if layout and layout.strip("/") != target.strip("/"):
+            source, place = layout.strip("/") + "/", target.strip("/") + "/"
+            members = {
+                place + dest[len(source):]: sha1
+                for dest, sha1 in members.items() if dest.startswith(source)
+            }
         if not members:
             expanded.append(entry)
             continue
         inherited = {
             key: value
             for key, value in entry.items()
-            if key not in ("type", "standalone_path")
+            if key not in ("type", "standalone_path", "layout_path")
         }
         expanded.extend(
             {**inherited, "name": dest, "path": dest, "sha1": sha1}
             for dest, sha1 in members.items()
-            if dest not in named
+            if (entry.get("root") or "", dest) not in named
         )
     return expanded
 
@@ -2905,6 +2927,32 @@ def own_build_tags(config: dict, standalone: set[str]) -> set[str]:
     }
 
 
+def twin_build_names(config: dict) -> set[str]:
+    """The libretro core names of emulators a platform also runs standalone
+    under another name.
+
+    RetroBat's es_systems.cfg offers MAME as the libretro core mame and as
+    the standalone mame64, two names of one profile: the platform runs it
+    both ways, and each build reads its own layout.
+    """
+    standalone = {str(c) for c in config.get("standalone_cores") or []}
+    libretro = {str(c) for c in config.get("cores") or []} - standalone if isinstance(
+        config.get("cores"), list
+    ) else set()
+    if not standalone or not libretro:
+        return set()
+    profiles = load_emulator_profiles(str(Path(__file__).resolve().parent.parent / "emulators"))
+    found: set[str] = set()
+    for key, profile in profiles.items():
+        kind = str(profile.get("type", ""))
+        if "libretro" not in kind or "standalone" not in kind:
+            continue
+        names = {key, *(str(c) for c in profile.get("cores") or [])}
+        if names & standalone:
+            found |= names & libretro
+    return found
+
+
 def standalone_set(config: dict) -> set[str]:
     """The emulators a platform launches standalone, with its dual-mode ones.
 
@@ -2924,6 +2972,7 @@ def standalone_set(config: dict) -> set[str]:
         & set(map(str, config.get("standalone_cores") or []))
     }
     names |= own_build_tags(config, names)
+    names |= {DUAL_MODE_PREFIX + n for n in twin_build_names(config)}
     # The system directory each core is handed travels the same way, so
     # platform_layouts places a core's files below it for every reader.
     names |= {
@@ -3157,6 +3206,29 @@ def standalone_bindings_for(
     )
 
 
+def _placed_entry(entry: dict, profile: dict, placed: tuple[str, str]) -> dict:
+    """A standalone entry moved to the (tree, path) a platform reads it from."""
+    tree, path = placed
+    moved = {**entry, "standalone_path": path,
+             "layout_path": standalone_layout_path(entry, profile)}
+    if entry.get("load_from"):
+        # The build's own root no longer applies: RetroDECK hands
+        # Cemu's MLC, read from its data directory, to bios/cemu.
+        load_from = entry["load_from"]
+        moved["load_from"] = {
+            **(load_from if isinstance(load_from, dict) else {"libretro": load_from}),
+            "standalone": "system_dir",
+        }
+    if tree == "install":
+        # Below the roms root inside the install tree (AmberELEC's
+        # /storage/roms): a plain dest the platform prefix skips.
+        return {**moved, "unprefixed": True}
+    # A file the launcher runs from the roms directory (ROCKNIX's
+    # roms/pico-8) or an emulator folder (RetroBat's emulators/pico8)
+    # is relative to it, and says so.
+    return {**moved, "root": tree} if tree else moved
+
+
 def platform_layouts(
     emu_name: str, profile: dict, standalone_cores: set[str]
 ) -> list[tuple[dict, bool]]:
@@ -3191,7 +3263,14 @@ def platform_layouts(
             for system in own:
                 family.setdefault(system, set()).update(entry_launch_systems(f, index) - own)
 
-    def bindings_of(entry: dict) -> dict[str, tuple[str, str]]:
+    # The systems the launcher records name: a launch for one of them reads
+    # the folder its record gives, every other launch the folder the records
+    # naming none give.
+    named_systems = frozenset().union(*(
+        h.systems for h in _handed(standalone_cores) if h.emulator in names
+    )) if per_system else frozenset()
+
+    def launch_systems(entry: dict) -> frozenset[str]:
         systems: frozenset[str] = frozenset()
         if per_system:
             own = entry_launch_systems(entry, {})
@@ -3205,12 +3284,53 @@ def platform_layouts(
                 systems = systems.union(*(
                     index.get(MACHINE_KEY + _norm_system_id(s), set()) for s in own
                 ))
+        return systems
+
+    def bindings_for(systems: frozenset[str]) -> dict[str, tuple[str, str]]:
         if systems not in by_systems:
             by_systems[systems] = standalone_bindings_for(
                 emu_name, profile, standalone_cores, systems
             )
         return by_systems[systems]
 
+    def bindings_of(entry: dict) -> dict[str, tuple[str, str]]:
+        return bindings_for(launch_systems(entry))
+
+    # The layout folders a search setting stands for (standalone_pools):
+    # bound, the launcher passes the setting and the default is not read.
+    pool_folders = {
+        str(v).strip("/") for v in (profile.get("standalone_pools") or {}).values()
+    } - {""}
+
+    def passed_setting(entry: dict) -> bool:
+        layout = str(standalone_layout_path(entry, profile) or "")
+        bindings = bindings_of(entry)
+        return any(
+            folder in bindings and (layout == folder or layout.startswith(folder + "/"))
+            for folder in pool_folders
+        )
+
+    def destinations(entry: dict) -> list[tuple[str, str] | None]:
+        """Each place the entry's launches read it from, in launch order.
+
+        An entry of a system a record names is read where that record puts
+        it. One also launched for systems no record names is read, for those
+        launches, where the records naming none put it: EmuELEC starts
+        AdvanceMAME with dir_rom roms/arcade for arcade and roms/mame for
+        mame and cps1-3 (advmame.sh:26-30), and an arcade set is read in both.
+        """
+        systems = launch_systems(entry)
+        found = [standalone_destination(entry, profile, bindings_for(systems))]
+        # A system the platform's list or machines put the entry under
+        # stands for the entry's own ids: XRoar's coco3 is launched as coco.
+        own = entry_launch_systems(entry, {})
+        launched = (systems - own) | (own & named_systems) if systems != own else systems
+        if launched & named_systems and launched - named_systems:
+            generic = standalone_destination(entry, profile, bindings_for(frozenset()))
+            if generic not in found:
+                found.append(generic)
+        return found
+
     layout_entries = [
         f for f in profile.get("files") or []
         if isinstance(f, dict) and standalone_layout_path(f, profile) is not None
@@ -3228,30 +3348,18 @@ def platform_layouts(
             if not isinstance(f, dict) or standalone_layout_path(f, profile) is None:
                 files.append(f)
                 continue
-            placed = standalone_destination(f, profile, bindings_of(f))
-            if placed is None or placed[0] == "outside":
+            places = [p for p in destinations(f) if p is not None and p[0] != "outside"]
+            if not places:
                 files.append({**f, "unplaced": OUTSIDE_BIOS_FOLDER})
                 continue
-            tree, path = placed
-            moved = {**f, "standalone_path": path,
-                     "layout_path": standalone_layout_path(f, profile)}
-            if f.get("load_from"):
-                # The build's own root no longer applies: RetroDECK hands
-                # Cemu's MLC, read from its data directory, to bios/cemu.
-                load_from = f["load_from"]
-                moved["load_from"] = {
-                    **(load_from if isinstance(load_from, dict) else {"libretro": load_from}),
-                    "standalone": "system_dir",
-                }
-            if tree == "install":
-                # Below the roms root inside the install tree (AmberELEC's
-                # /storage/roms): a plain dest the platform prefix skips.
-                files.append({**moved, "unprefixed": True})
-                continue
-            # A file the launcher runs from the roms directory (ROCKNIX's
-            # roms/pico-8) or an emulator folder (RetroBat's emulators/pico8)
-            # is relative to it, and says so.
-            files.append({**moved, "root": tree} if tree else moved)
+            moved = f
+            if f.get("bundled") and passed_setting(f):
+                # The copy installed with the emulator sits where the setting
+                # pointed before the launcher moved it: RetroBat passes MAME
+                # -hash bios/mame/hash (Mame64.Generator.cs:172, 177-178), and
+                # the hash/ beside mame.exe is no longer read.
+                moved = {k: v for k, v in f.items() if k != "bundled"}
+            files.extend(_placed_entry(moved, profile, placed) for placed in places)
         standalone = {**profile, "files": files}
     dual = {c.removeprefix(DUAL_MODE_PREFIX) for c in standalone_cores
             if c.startswith(DUAL_MODE_PREFIX)}

scripts/launchers.py

@@ -314,8 +314,9 @@ BIOS_ROOT_VARIABLES = ("$biosdir", "$home/RetroPie/BIOS", "${biosPath}", "$bios_
 
 # The name a platform's launch code gives its roms root: RetroDECK's
 # component scripts write $roms_path (retrodeck_launchers), the setting whose
-# default is the roms root (config/retrodeck/retrodeck.json:5).
-ROMS_ROOT_VARIABLES = ("$roms_path", "${roms_path}")
+# default is the roms root (config/retrodeck/retrodeck.json:5), RetroPie's
+# scriptmodules $romdir (retropie_packages.sh:47-49).
+ROMS_ROOT_VARIABLES = ("$roms_path", "${roms_path}", "$romdir", "${romdir}")
 
 # The kinds that hand an emulator a whole directory. A scans_dir record sets
 # the directory the emulator picks its file from: ROCKNIX writes ARMSX2's

scripts/packextras.py

@@ -1545,6 +1545,11 @@ def _collect_emulator_extras(
                     extra["system_dir"] = f["system_dir"]
                 if f.get("unprefixed"):
                     extra["unprefixed"] = True
+                # A copy placed outside the BIOS tree resolves by its layout
+                # path alone, as the first pass does: EmuELEC's
+                # roms/mame/stvbios.zip took bios/Arcade/MAME's set by that tail.
+                if f.get("layout_path"):
+                    extra["layout_path"] = f["layout_path"]
                 extra["claim_order"] = (2, 0, emu_name)
                 if _take(extra, full_dest, seen_dests, taken):
                     extras.append(extra)

scripts/scraper/_hash_merge.py

@@ -578,8 +578,10 @@ def _append_new_entries(text: str, files: list[dict], original: str) -> str:
         return text
 
     lines = []
+    profile_type = str(existing_data.get("type") or "")
     for fe in new_entries:
         lines.append(f"\n  - name: {_yaml_scalar(fe['name'])}")
+        lines.extend(set_path_lines(profile_type, fe["name"]))
         lines.append(f"    required: {str(fe['required']).lower()}")
         lines.append("    category: bios_zip")
         if fe.get("source_ref"):
@@ -715,10 +717,27 @@ def _rewrite_systems(section: str, systems: dict[str, list[str]]) -> str:
     return "\n".join(out)
 
 
-def _format_device_entry(name: str, device: dict, role: str) -> str:
+def set_path_lines(profile_type: str, name: str) -> list[str]:
+    """Where each build of a MAME profile reads a set it lists.
+
+    The libretro build searches the content folder, then <system>/mame/bios
+    and <system>/mame/roms (libretro-internal/retro_init.cpp,
+    Set_Path_Option), never the system folder itself. The standalone build
+    reads the top of its rompath.
+    """
+    if "libretro" not in profile_type:
+        return []
+    lines = [f'    path: "mame/bios/{name}"']
+    if "standalone" in profile_type:
+        lines.append(f'    standalone_path: "{name}"')
+    return lines
+
+
+def _format_device_entry(name: str, device: dict, role: str, profile_type: str = "") -> str:
     """One generated device entry, in the field order of the written ones."""
     lines = [
         f"  - name: {_zip_scalar(name)}",
+        *set_path_lines(profile_type, name),
         "    required: false",
         "    category: bios_zip",
         *_system_lines(device.get("systems") or []),
@@ -779,7 +798,9 @@ def merge_mame_devices(
         name = f"{set_name}.zip"
         existing = entries.get(name)
         if not existing:
-            new_blocks[name] = _format_device_entry(name, device, role)
+            new_blocks[name] = _format_device_entry(
+                name, device, role, str(profile.get("type") or "")
+            )
             report["added"].append(set_name)
             continue
         if any(e.get("category") != "bios_zip" for e in existing):

scripts/scraper/batocera_launchers.py

@@ -1119,10 +1119,32 @@ def patched_paths(read: Callable[[str], str], patches: tuple[str, ...] = PATH_PA
     return launchers
 
 
+# Generators whose command line hands the emulator /userdata/bios folders as
+# search paths: MAME's -rompath and -samplepath (mame/mameGenerator.py).
+SEARCH_PATH_GENERATORS = ("mame/mameGenerator.py",)
+
+
+def search_paths(cg: Configgen) -> list[Launcher]:
+    """Every /userdata/bios path a generator's command line names, read as
+    KNULLI's generic reader reads its fork of the same generator."""
+    try:
+        from .knulli_launchers import GenericSites, bios_sites  # noqa: PLC0415
+    except ImportError:
+        from knulli_launchers import GenericSites, bios_sites  # type: ignore[no-redef]  # noqa: PLC0415
+    # A boot disk image the command line names (the Macintosh's
+    # /userdata/bios/<boot_disk>.img) is a file, no search path.
+    return [
+        launcher
+        for path in SEARCH_PATH_GENERATORS
+        for launcher in bios_sites(cg, GenericSites(path, "passes_path"))
+        if re.match(r"command line -\w", launcher.passes_to or "")
+    ]
+
+
 EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = (
     duckstation, duckstation_legacy, hatari, xemu, melonds, mupen64plus,
     x16emu, gsplus, lexaloffle, environment_directories, config_directories,
-    core_file_options, working_directories,
+    core_file_options, working_directories, search_paths,
 )
 
 

scripts/scraper/mame_hash_scraper.py

@@ -453,6 +453,10 @@ def release_profile(
         name = f"{set_name}.zip"
         entry: dict[str, Any] = {
             "name": name,
+            # The libretro build searches the content folder, then
+            # <system>/mame/bios and <system>/mame/roms (retro_init.cpp,
+            # Set_Path_Option): the system folder itself is not read.
+            "path": f"mame/bios/{name}",
             "required": settled_required(known.get(name, {}), data),
             "category": "bios_zip",
         }

scripts/scraper/retropie_launchers.py

@@ -32,6 +32,9 @@ SECTIONS = ("emulators", "libretrocores", "ports")
 PACKAGES_SCRIPT = "retropie_packages.sh"
 
 BIOSDIR = "$biosdir"
+# The roms root (retropie_packages.sh:47-49): MAME's mame.ini lists rom,
+# sample and artwork folders below it (scriptmodules/emulators/mame.sh:133-136).
+ROMDIR = "$romdir"
 # The package whose config every lr- core runs under.
 LIBRETRO_FRONTEND = "retroarch"
 # The launch scripts copied next to an emulator spell the directory out.
@@ -464,6 +467,20 @@ class Scripts:
         return results
 
 
+def _search_list_in_roms(event: "Event") -> bool:
+    """An iniSet whose value is a list of folders below the roms root.
+
+    One folder alone is where a program writes (cfg_directory, nvram_directory
+    and the others MAME's mame.ini sets): a list is where it searches.
+    """
+    if len(event.words) < 3 or len(event.words[2]) != 1:
+        return False
+    parts = [p for p in event.words[2][0].split(";") if p]
+    return len(parts) > 1 and all(
+        p.startswith(ROMDIR + "/") or p.startswith("${romdir}/") for p in parts
+    )
+
+
 def _mentions_bios(text: str) -> bool:
     return BIOSDIR in text or "${biosdir}" in text or BIOS_SPELLED in text
 
@@ -541,6 +558,12 @@ class PackageReader:
         if command == "addEmulator":
             self._add_emulator(event)
             return
+        if command == "iniSet" and not _mentions_bios(event.raw):
+            # A search path below the roms root: MAME reads its samples from
+            # samplepath $romdir/mame/samples;$romdir/arcade/samples.
+            if _search_list_in_roms(event):
+                self._ini_set(event)
+            return
         if not _mentions_bios(event.raw):
             return
         if command in HOUSEKEEPING: