chore: merge the MAME ES-DE search paths

scripts/common.py

@@ -2872,9 +2872,10 @@ CONTENT_SCOPE_TAG = "contentscope:"
 # [core name, roms folder, (tree, path)]: a libretro core the platform
 # launches for the games of that folder, which it reads beside.
 CONTENT_FOLDER_PREFIX = "contentdir:"
-# [emulator, (tree, path), system, machines, any machine, source_ref]: a
-# standalone emulator handed a roms folder as its ROM search path on the
-# command line of a system's launch.
+# [emulator or core, (tree, path), system, machines, any machine,
+# source_ref, build]: a standalone emulator, or a libretro core in the
+# command line it is handed as content, given a roms folder as its ROM
+# search path by a system's launch command.
 ROM_SEARCH_PREFIX = "romsearch:"
 # The frontend writes the BIOS it hands a core beside the content: the
 # folder it gives the BIOS is the content's.
@@ -3413,15 +3414,16 @@ def _core_key(name: str) -> str:
 
 def content_folder_tags(config: dict) -> set[str]:
     """The roms folders a platform launches each libretro core for, and the
-    ROM search paths its commands hand standalone emulators, as tags.
+    ROM search paths its commands hand emulators and cores, as tags.
 
     A core reading its BIOS sets beside the game (MAME 2003's osd_get_path
     gives FILETYPE_ROM the content directory) reads them in the folder of
     the system it is launched for. The folders come from the platform's own
-    lists: ``libretro_systems``, the default core of each folder its rom
-    folders file records, the cores a system offers under its own name, and
-    the systems a RetroPie package registers. A platform with no roms root,
-    or one keeping no folder per system (RetroArch, Lakka), gives none.
+    lists: ``libretro_systems``, every core a folder of its rom folders file
+    offers (its default, its board defaults, ``offers``), the cores a system
+    offers under its own name, and the systems a RetroPie package registers.
+    A platform with no roms root, or one keeping no folder per system
+    (RetroArch, Lakka), gives none.
     """
     if config.get("inherits_from") == "emulatorjs":
         # EmulatorJS writes the BIOS a frontend hands it and the game to
@@ -3437,7 +3439,8 @@ def content_folder_tags(config: dict) -> set[str]:
     for core, systems in (config.get("libretro_systems") or {}).items():
         pairs |= {(_core_key(core), str(s)) for s in systems or []}
     for row in rows:
-        for choice in [row, *(row.get("board_defaults") or {}).values()]:
+        choices = [row, *(row.get("board_defaults") or {}).values(), *(row.get("offers") or [])]
+        for choice in choices:
             if choice.get("core") and choice.get("emulator") in LIBRETRO_RUNNERS:
                 pairs.add((_core_key(choice["core"]), str(row["folder"])))
     for system in (config.get("systems") or {}).values():
@@ -3466,9 +3469,10 @@ def content_folder_tags(config: dict) -> set[str]:
             for read in search.get("reads") or []:
                 if read in folders:
                     tags.add(ROM_SEARCH_PREFIX + json.dumps([
-                        str(search["emulator"]), place(read), str(row["folder"]),
-                        sorted(search.get("machines") or []), bool(search.get("any_machine")),
-                        str(search.get("source_ref", "")),
+                        str(search.get("core") or search["emulator"]), place(read),
+                        str(row["folder"]), sorted(search.get("machines") or []),
+                        bool(search.get("any_machine")), str(search.get("source_ref", "")),
+                        "libretro" if search.get("core") else "standalone",
                     ]))
     return tags
 
@@ -3485,6 +3489,7 @@ ANY_MACHINE_POPULATIONS: dict[str, tuple[str, tuple[str, ...]]] = {
     "cps2": ("source", ("capcom/cps2.cpp",)),
     "cps3": ("source", ("capcom/cps3.cpp",)),
     "model2": ("source", ("sega/model2.cpp",)),
+    "model3": ("source", ("sega/model3.cpp",)),
     "daphne": ("device_source", ("devices/machine/ld",)),
     "laserdisc": ("device_source", ("devices/machine/ld",)),
     "gameandwatch": ("source", ("handheld/",)),
@@ -3550,10 +3555,11 @@ def content_layout(
 
     A libretro entry read beside the content (``load_from: content_dir``)
     goes to the folder of each system the platform launches the core for,
-    and is named as read outside the BIOS folder where it knows none. A
-    standalone entry no launcher record places, under the root of the
-    layout a passed ROM search path stands for, goes to each folder a
-    command passes for a machine that reads it. Outside a platform (no
+    and is named as read outside the BIOS folder where it knows none. An
+    entry in the folder a passed ROM search path stands for (the root of a
+    standalone layout, a ``libretro_pools`` folder) also goes to each folder
+    a command passes for a machine that reads it, and a standalone entry no
+    launcher record places goes there only. Outside a platform (no
     CONTENT_SCOPE_TAG) the view is left as it is.
     """
     tags = [t for t in standalone_cores if t.startswith(CONTENT_TAG_PREFIXES)]
@@ -3571,6 +3577,29 @@ def content_layout(
     files = view.get("files") or []
     out: list = []
     changed = False
+    searches = [
+        json.loads(t[len(ROM_SEARCH_PREFIX):])
+        for t in tags if t.startswith(ROM_SEARCH_PREFIX)
+    ]
+    build = "standalone" if standalone else "libretro"
+    searches = [s for s in searches if _core_key(s[0]) in names and s[6] == build]
+
+    def search_reads(name: str) -> list[tuple[str, str]]:
+        stem = name.rsplit(".", 1)[0]
+        return sorted({
+            tuple(where) for _emu, where, system, machines, any_machine, _ref, _build in searches
+            if where and stem in rom_search_sets(emu_name, system, machines, any_machine)
+        })
+
+    def roms_entry(entry: dict, tree: str, folder: str, rel: str) -> dict:
+        # Below the roms root inside the install tree (EmuELEC's
+        # /storage/roms): a plain dest the platform prefix skips, as a
+        # standalone entry placed there (_placed_entry).
+        where: dict[str, object] = (
+            {"unprefixed": True} if tree == "install" else {"root": ROMS_ROOT}
+        )
+        return {**entry, **where, "path": f"{folder}/{rel}", "layout_path": rel}
+
     if not standalone:
         folders = sorted({
             tuple(where) for core, _folder, where in (
@@ -3578,11 +3607,25 @@ def content_layout(
                 for t in tags if t.startswith(CONTENT_FOLDER_PREFIX)
             ) if core in names and where
         })
+        # The folders a passed -rompath replaces the build's own search
+        # path with stand for these: libretro MAME hands MAME
+        # <system>/mame/bios unless the content's command line names
+        # another (retro_init.cpp Set_Path_Option, execute_game_cmd).
+        from launchers import pool_folders  # noqa: PLC0415
+
+        pools = set(pool_folders({"standalone_pools": view.get("libretro_pools")}).values())
         for f in files:
-            if (not isinstance(f, dict) or _content_load(f) != "content_dir"
-                    or f.get("category") == "game_data"):
+            if not isinstance(f, dict):
+                out.append(f)
+                continue
+            if _content_load(f) != "content_dir" or f.get("category") == "game_data":
                 # Game data read beside the content is the game's own.
                 out.append(f)
+                parent, _, rel = str(f.get("path") or "").rpartition("/")
+                if searches and parent in pools and rel:
+                    reads = search_reads(rel)
+                    changed |= bool(reads)
+                    out.extend(roms_entry(f, tree, folder, rel) for tree, folder in reads)
                 continue
             changed = True
             plain = _without_libretro_load(f)
@@ -3590,33 +3633,33 @@ def content_layout(
                 out.append({**plain, "unplaced": OUTSIDE_BIOS_FOLDER})
                 continue
             rel = str(f.get("path") or f.get("name") or "")
-            for tree, folder in folders:
-                # Below the roms root inside the install tree (EmuELEC's
-                # /storage/roms): a plain dest the platform prefix skips, as
-                # a standalone entry placed there (_placed_entry).
-                where: dict[str, object] = (
-                    {"unprefixed": True} if tree == "install" else {"root": ROMS_ROOT}
-                )
-                out.append({**plain, **where, "path": f"{folder}/{rel}", "layout_path": rel})
+            out.extend(roms_entry(plain, tree, folder, rel) for tree, folder in folders)
         return {**view, "files": out} if changed else view
-    searches = [
-        json.loads(t[len(ROM_SEARCH_PREFIX):])
-        for t in tags if t.startswith(ROM_SEARCH_PREFIX)
-    ]
-    searches = [s for s in searches if _core_key(s[0]) in names]
     pools = {str(v).strip("/") for v in (view.get("standalone_pools") or {}).values()}
     if not searches or "" not in pools:
         return view
     for f in files:
-        layout = standalone_layout_path(f, view) if isinstance(f, dict) else None
-        if not layout or "/" in layout or f.get("unplaced") != OUTSIDE_BIOS_FOLDER:
+        layout = (
+            (f.get("layout_path") or standalone_layout_path(f, view)) if isinstance(f, dict) else None
+        )
+        if not layout or "/" in layout:
             out.append(f)
             continue
-        stem = layout.rsplit(".", 1)[0]
-        reads = sorted({
-            tuple(where) for _emu, where, system, machines, any_machine, _ref in searches
-            if where and stem in rom_search_sets(emu_name, system, machines, any_machine)
-        })
+        reads = search_reads(layout)
+        if f.get("unplaced") != OUTSIDE_BIOS_FOLDER:
+            # A set a launcher record placed stays there: EmuDeck's
+            # mame.ini rompath is read when Steam ROM Manager starts MAME
+            # with no -rompath (tools/launchers/mame.sh), and ES-DE's
+            # commands read the folders they pass.
+            out.append(f)
+            held = {(f.get("root") or "", f.get("standalone_path") or "")}
+            extra = [
+                _placed_entry(f, view, (tree, f"{folder}/{layout}")) for tree, folder in reads
+            ]
+            extra = [e for e in extra if (e.get("root") or "", e["standalone_path"]) not in held]
+            changed |= bool(extra)
+            out.extend(extra)
+            continue
         if not reads:
             out.append(f)
             continue

scripts/packextras.py

@@ -10,7 +10,7 @@ from collections.abc import Iterable
 
 from common import PROFILE_IDENTITY_FIELDS, _norm_system_id, entry_system_ids
 from alternatives import core_file_spares, required_only_spares, set_fields
-from common import archive_members, holds_required_members
+from common import archive_members, contents_members, holds_required_members
 from common import declared_root, expand_platform_declared_names, platform_declarations
 from common import load_emulator_profiles
 from common import resolution_is_hash_exact
@@ -1447,6 +1447,8 @@ def _collect_emulator_extras(
     for e in extras:
         covered_names.add(e["name"])
 
+    # Destination -> the copies cores ask for there, in profile order.
+    copied: dict[str, list[dict]] = {}
     for emu_name, profile in sorted(profiles.items()):
         if profile.get("type") in ("launcher", "alias"):
             continue
@@ -1500,7 +1502,8 @@ def _collect_emulator_extras(
                 )
                 if full_dest in seen_dests and full_dest not in taken:
                     continue
-                if full_dest in taken and full_dest not in seen and fname not in platform_names:
+                if (full_dest in taken or full_dest in copied) and full_dest not in seen \
+                        and fname not in platform_names:
                     # Another core's extra holds this path and only that
                     # extra covers the name: a pack without that core never
                     # reaches this copy, so this core claims nothing here.
@@ -1551,8 +1554,22 @@ def _collect_emulator_extras(
                 if f.get("layout_path"):
                     extra["layout_path"] = f["layout_path"]
                 extra["claim_order"] = (2, 0, emu_name)
-                if _take(extra, full_dest, seen_dests, taken):
-                    extras.append(extra)
+                copied.setdefault(full_dest, []).append(extra)
+
+    # Several cores reading one archive in one roms folder: the copy of the
+    # core the folder starts by default goes first, then the copy serving
+    # the most of them. Five cores AmberELEC offers for roms/arcade read
+    # neogeo.zip there, and the copy holding the 18 members its default
+    # MAME 2003-Plus requires also holds the 3 MAME 2000 requires.
+    defaults = _folder_defaults(config, profiles)
+    for full_dest, claims in copied.items():
+        if len(claims) > 1:
+            claims = sorted(claims, key=lambda c: (
+                not _starts_by_default(c, defaults), -_copy_serves(c, claims, db),
+            ))
+        for extra in claims:
+            if _take(extra, full_dest, seen_dests, taken):
+                extras.append(extra)
 
     extras.extend(
         _archive_prefix_extras(
@@ -1656,6 +1673,42 @@ class _ClaimRanks:
         )
 
 
+def _folder_defaults(config: dict, profiles: dict) -> dict[str, set[str]]:
+    """Roms folder -> the profiles its default core or emulator names."""
+    out: dict[str, set[str]] = {}
+    for row in config.get("_rom_folders") or []:
+        chosen = {str(row.get("core") or ""), str(row.get("emulator") or "")} - {""}
+        keys = {c.lower().replace("-", "_") for c in chosen}
+        out[str(row.get("folder"))] = {
+            name for name, profile in profiles.items()
+            if keys & {n.lower().replace("-", "_")
+                       for n in (name, *(str(c) for c in profile.get("cores") or []))}
+        }
+    return out
+
+
+def _starts_by_default(extra: dict, defaults: dict[str, set[str]]) -> bool:
+    """Whether the extra's core is the one its roms folder starts by default."""
+    layout = str(extra.get("layout_path") or "")
+    destination = str(extra.get("destination") or "")
+    if not layout or not destination.endswith("/" + layout):
+        return False
+    folder = destination[: -len(layout) - 1].removeprefix("roms/")
+    return extra.get("source_profile") in defaults.get(folder, set())
+
+
+def _copy_serves(extra: dict, claims: list[dict], db: dict) -> int:
+    """How many of the claims on one path the copy an extra resolves to
+    holds every required member of."""
+    wanted = [members for c in claims if (members := contents_members(c))]
+    if not wanted:
+        return 0
+    path, _status = resolve_local_file(extra, db, dest_hint=extra.get("destination", ""))
+    if not path:
+        return 0
+    return sum(1 for members in wanted if holds_required_members(path, members))
+
+
 def _names_content(entry: dict) -> bool:
     """Whether a report entry declares the bytes it wants, not only a name.
 

scripts/scraper/romfolders/__init__.py

@@ -36,7 +36,7 @@ SETUP_KEYS = (
 )
 ROW_KEYS = (
     "folder", "name", "aliases", "extensions", "files", "core", "emulator", "source_ref",
-    "board_defaults", "arcade", "searched_by", "note",
+    "board_defaults", "offers", "arcade", "searched_by", "note",
 )
 
 

scripts/scraper/romfolders/es.py

@@ -46,6 +46,44 @@ def _board(core: str | None, emulator: str | None, source_ref: str) -> dict:
     return {k: str(v) for k, v in out.items() if v}
 
 
+def _offers(choices: list[dict], core: str | None, emulator: str | None) -> list[dict]:
+    """Every (emulator, core) a folder can start, in the list's order, or
+    none when the default is the only one.
+
+    A core written under several branches of a device case keeps the
+    conditions it is written under, and none once a branch is the `*)` one
+    beside another or no branch at all: every build then offers it.
+    """
+    found: dict[tuple[str, str], dict] = {}
+    whens: dict[tuple[str, str], set[str]] = {}
+    lines: dict[tuple[str, str], list[str]] = {}
+    for choice in choices:
+        pair = (str(choice.get("emulator") or ""), str(choice.get("core") or choice.get("emulator") or ""))
+        if not pair[1]:
+            continue
+        whens.setdefault(pair, set()).add(str(choice.get("when") or ""))
+        ref = str(choice.get("source_ref") or "")
+        if ref and pair in found and found[pair].get("source_ref", "").split(":")[0] == ref.split(":")[0]:
+            lines[pair].append(ref.rsplit(":", 1)[-1])
+        if pair not in found:
+            found[pair] = {k: str(v) for k, v in {**choice, "core": pair[1]}.items()
+                           if v and k != "when"}
+            lines[pair] = []
+    out = []
+    for pair, offer in found.items():
+        if lines[pair] and offer.get("source_ref"):
+            offer["source_ref"] = ",".join([offer["source_ref"], *lines[pair]])
+        conditions = whens[pair]
+        if "" not in conditions and not (
+                len(conditions) > 1 and any(w.endswith(" *") for w in conditions)):
+            offer = {**offer, "when": "; ".join(sorted(conditions))}
+            offer = {k: offer[k] for k in ("emulator", "core", "when", "source_ref") if k in offer}
+        out.append(offer)
+    if [(o.get("emulator", ""), o["core"]) for o in out] == [(emulator or "", core or emulator or "")]:
+        return []
+    return out
+
+
 def _top_key_lines(text: str) -> dict[str, int]:
     """First line of each top-level YAML key."""
     out: dict[str, int] = {}
@@ -286,6 +324,9 @@ def _xml_rows(text: str, key: str, path: str, params: dict) -> list[dict]:
             "core": core or emulator,
             "emulator": emulator,
             "source_ref": _ref(key, path, row["line"]),
+            "offers": _offers(
+                [{"emulator": e, "core": c} for e, cores in emulators for c, _d in cores or [(e, False)]],
+                core, emulator),
             "arcade": _is_arcade(system.findtext("hardware"), platform),
             "note": row.get("note"),
         })
@@ -322,6 +363,9 @@ def es_json(spec: dict, fetcher: Fetcher) -> list[dict]:
             "core": core or emulator,
             "emulator": emulator,
             "source_ref": _ref(key, path, row["line"]),
+            "offers": _offers(
+                [{"emulator": e, "core": c} for e, cores in emulators for c, _d in cores or [(e, False)]],
+                core, emulator),
             "arcade": _is_arcade(system.get("hardware"), system.get("platform")),
             "note": row.get("note"),
         })
@@ -470,6 +514,12 @@ def elec_conf(spec: dict, fetcher: Fetcher) -> list[dict]:
             "emulator": pick["emulator"] if pick else None,
             "source_ref": "; ".join(refs),
             "board_defaults": board_defaults,
+            "offers": _offers([
+                {"emulator": c["emulator"], "core": c["core"],
+                 "when": ", ".join(f"{v} {p}".strip() for v, p in c["branches"]),
+                 "source_ref": _ref(key, package, c["line"])}
+                for c in mine
+            ], pick and pick["core"], pick and pick["emulator"]),
             "arcade": values.get("SYSTEM_HARDWARE") == "arcade" or None,
             "note": note,
         })
@@ -526,21 +576,55 @@ def _alternative_emulators(text: str, function: str) -> dict[str, tuple[str, int
 _ROMPATH_OPTIONS = ("-rompath", "-rp")
 
 
+def _core_words(command: str) -> tuple[str, list[str]] | None:
+    """The core and the words of the content RetroArch hands it.
+
+    ES-DE starts the libretro MAME core with a quoted command line as its
+    content (`-L .../mame_libretro.so "apple2e -rompath \\"...\\" ..."`): popen
+    runs it through the shell (es-core/src/utils/PlatformUtil.cpp:231, 257),
+    and the core splits the content on spaces and quotes when it is no file
+    (retro_init.cpp parse_cmdline, mmain2).
+    """
+    core = _RA_CORE.search(command)
+    if core is None:
+        return None
+    try:
+        args = shlex.split(command)
+        at = args.index("-L")
+        content = shlex.split(args[at + 2]) if at + 2 < len(args) else []
+    except ValueError:
+        return None
+    return core.group(1), content
+
+
 def _searched_by(commands: list[tuple[str, str, int | None]], key: str, path: str) -> list[dict]:
-    """The ROM folders each standalone emulator's commands pass it, by emulator.
+    """The ROM folders each emulator's commands pass it, by emulator and core.
 
     ES-DE starts standalone MAME with `-rompath %GAMEDIR%\\;%ROMPATH%/<system>`
     and a machine, or %BASENAME% where the game is the machine. The command
-    line outranks mame.ini. %GAMEDIR% is the game's own folder, no folder a
-    pack fills.
+    line outranks mame.ini. The libretro MAME core gets the same option in
+    the command line it is handed as content, after the machine. %GAMEDIR%
+    is the game's own folder, no folder a pack fills.
     """
-    found: dict[str, dict] = {}
+    found: dict[tuple[str, str], dict] = {}
     for _label, command, line in commands:
-        emulator = _EMULATOR.search(command)
-        if emulator is None or _RA_CORE.search(command):
-            continue
-        words = command.replace("\\;", ";").split()
-        at = next((i for i, w in enumerate(words) if w in _ROMPATH_OPTIONS), None)
+        libretro = _core_words(command)
+        if libretro is not None:
+            core, words = libretro
+            if words[:1] == [core]:
+                # A first word naming the core is dropped (mmain2).
+                words = words[1:]
+            name = "retroarch"
+            at = next((i for i, w in enumerate(words) if w in _ROMPATH_OPTIONS), None)
+            machine = words[0] if words and at else ""
+        else:
+            emulator = _EMULATOR.search(command)
+            if emulator is None:
+                continue
+            core, name = "", emulator.group(1).lower()
+            words = command.replace("\\;", ";").split()
+            at = next((i for i, w in enumerate(words) if w in _ROMPATH_OPTIONS), None)
+            machine = words[at + 2] if at is not None and at + 2 < len(words) else ""
         if at is None or at + 1 >= len(words):
             continue
         reads = [
@@ -549,11 +633,10 @@ def _searched_by(commands: list[tuple[str, str, int | None]], key: str, path: st
         ]
         if not reads:
             continue
-        name = emulator.group(1).lower()
-        record = found.setdefault(name, {"emulator": name, "reads": [], "machines": [],
-                                         "any_machine": False, "lines": []})
+        record = found.setdefault((name, core), {"emulator": name, "core": core, "reads": [],
+                                                 "machines": [], "any_machine": False,
+                                                 "lines": []})
         record["reads"] += [r for r in reads if r not in record["reads"]]
-        machine = words[at + 2] if at + 2 < len(words) else ""
         if not machine or machine.startswith("%"):
             record["any_machine"] = True
         elif machine not in record["machines"]:
@@ -561,7 +644,10 @@ def _searched_by(commands: list[tuple[str, str, int | None]], key: str, path: st
         record["lines"].append(line)
     rows = []
     for record in found.values():
-        row = {"emulator": record["emulator"], "reads": record["reads"]}
+        row = {"emulator": record["emulator"]}
+        if record["core"]:
+            row["core"] = record["core"]
+        row["reads"] = record["reads"]
         if record["machines"]:
             row["machines"] = record["machines"]
         if record["any_machine"]:
@@ -611,6 +697,7 @@ def esde(spec: dict, fetcher: Fetcher) -> list[dict]:
                 "folder": folder, "extensions": extensions, "commands": commands,
                 "ref": _ref(key, path, lines.get(name)),
                 "searched_by": _searched_by(placed, key, path),
+                "placed": [(*c, key, path) for c in placed],
             }
     alternatives: dict[str, tuple[str, int]] = {}
     alt = params.get("alternatives")
@@ -633,6 +720,11 @@ def esde(spec: dict, fetcher: Fetcher) -> list[dict]:
             else:
                 note = f"alternative emulator '{label}' names no command of this system"
         emulator, core = _command_core(chosen[1])
+        offers = _offers([
+            {"emulator": e, "core": c, "source_ref": _ref(key_, path_, line)}
+            for _label, command, line, key_, path_ in system["placed"]
+            for e, c in [_command_core(command)] if e
+        ], core, emulator)
         extensions = system["extensions"]
         if "." in extensions:
             # ES-DE's "." is a file without an extension (SystemData.cpp
@@ -647,6 +739,7 @@ def esde(spec: dict, fetcher: Fetcher) -> list[dict]:
             "emulator": emulator,
             "source_ref": "; ".join(refs),
             "note": note,
+            "offers": offers,
             "searched_by": system["searched_by"] or None,
         })
     return rows