chore: merge the last MAME fixes

scripts/common.py

@@ -3390,7 +3390,7 @@ def _build_layouts(
             if f.get("bundled") and passed_setting(f):
                 # The copy installed with the emulator sits where the setting
                 # pointed before the launcher moved it: RetroBat passes MAME
-                # `-hash` bios/mame/hash (Mame64.Generator.cs:172, 177-178), and
+                # -hash bios/mame/hash (Mame64.Generator.cs:172, 177-178), and
                 # the hash/ beside mame.exe is no longer read.
                 moved = {k: v for k, v in f.items() if k != "bundled"}
             files.extend(_placed_entry(moved, profile, placed) for placed in places)
@@ -3412,6 +3412,107 @@ def _core_key(name: str) -> str:
     return str(name).lower().replace("-", "_")
 
 
+# Set on a platform config built for one board (generate_pack, verify
+# --target): a core a roms folder offers on other boards only is not placed
+# there.
+TARGET_KEY = "_target"
+
+
+def _branch_holds(branch: str, board: str) -> bool:
+    """Whether one `VARIABLE PATTERN` branch of a device case admits a board.
+
+    ROCKNIX files a board under the DEVICE value its target scraper reads,
+    lowercased. A branch on any other variable (PREFER_GLES) is no board
+    condition, and admits every board.
+    """
+    variable, _, pattern = branch.strip().partition(" ")
+    if variable != "DEVICE" or not pattern.strip():
+        return True
+    import fnmatch  # noqa: PLC0415
+
+    return any(
+        fnmatch.fnmatchcase(board.upper(), p.strip().upper()) for p in pattern.split("|")
+    )
+
+
+def when_admits(when: str, board: str) -> bool:
+    """Whether an offer's `when` holds on a board: alternatives joined by
+    "; ", each the branches it is nested under joined by ", "."""
+    return any(
+        all(_branch_holds(branch, board) for branch in alternative.split(", "))
+        for alternative in when.split("; ")
+    )
+
+
+def offer_boards(config: dict) -> dict[tuple[str, str], frozenset[str] | None]:
+    """The boards on which each roms folder offers each core, by
+    (core key, folder), None where every board does.
+
+    ROCKNIX writes a core under a `case "${DEVICE}"` branch for the devices
+    it builds (`when`), and a default for a named device in board_defaults
+    (DEVICE=RK3326). Boards are the platform's targets, aliases included.
+    """
+    platform = str(config.get("_key") or "")
+    targets = list_available_targets(
+        platform, str(Path(__file__).resolve().parent.parent / "platforms")
+    ) if platform else []
+    names = {t["name"]: [t["name"], *t.get("aliases", [])] for t in targets}
+
+    def admitted(test: Callable[[str], bool]) -> set[str]:
+        return {n for board, spelled in names.items() if test(board) for n in spelled}
+
+    every = admitted(lambda _board: True)
+    found: dict[tuple[str, str], set[str]] = {}
+    for row in config.get("_rom_folders") or []:
+        folder = str(row.get("folder") or "")
+        offers = row.get("offers") or []
+        offered = {(o.get("emulator"), _core_key(o.get("core") or "")) for o in offers}
+        choices: list[tuple[dict, set[str]]] = []
+        if row.get("core") and (row.get("emulator"), _core_key(row["core"])) not in offered:
+            choices.append((row, every))
+        for key, spec in (row.get("board_defaults") or {}).items():
+            branch = str(key).replace("=", " ", 1)
+            choices.append((spec, admitted(lambda b, branch=branch: _branch_holds(branch, b))))
+        for offer in offers:
+            when = str(offer.get("when") or "")
+            choices.append((offer, admitted(lambda b, when=when: when_admits(when, b))
+                            if when else every))
+        for choice, boards in choices:
+            if choice.get("core") and choice.get("emulator") in LIBRETRO_RUNNERS:
+                found.setdefault((_core_key(choice["core"]), folder), set()).update(boards)
+    return {
+        pair: None if not names or boards >= every else frozenset(boards)
+        for pair, boards in found.items()
+    }
+
+
+def content_folder_boards(config: dict) -> dict[tuple[str, str], frozenset[str] | None]:
+    """Each (core key, roms folder) a platform launches a libretro core for,
+    with the boards it does so on, None for every board."""
+    pairs: set[tuple[str, str]] = set()
+    for core, systems in (config.get("libretro_systems") or {}).items():
+        pairs |= {(_core_key(core), str(s)) for s in systems or []}
+    for system in (config.get("systems") or {}).values():
+        if isinstance(system, dict) and system.get("native_id"):
+            pairs |= {(_core_key(c), str(system["native_id"])) for c in system.get("cores") or []}
+    for record in config.get("launchers") or []:
+        if record.get("kind") == "package" and record.get("build") == "libretro":
+            package = str(record.get("emulator", ""))
+            if package.startswith("lr-mess"):
+                # Built with SUBTARGET=mess (lr-mess2016.sh:31): the MESS
+                # drivers alone, none of the arcade sets of the core it
+                # shares a repository with.
+                continue
+            core = _core_key(package.removeprefix("lr-"))
+            pairs |= {(core, str(s)) for s in record.get("systems") or []}
+    # Pairs from the platform's own lists hold on every board, an offer on
+    # the boards its frontend writes it for.
+    return {
+        **{pair: on for pair, on in offer_boards(config).items() if pair not in pairs},
+        **{pair: None for pair in pairs},
+    }
+
+
 def content_folder_tags(config: dict) -> set[str]:
     """The roms folders a platform launches each libretro core for, and the
     ROM search paths its commands hand emulators and cores, as tags.
@@ -3435,34 +3536,22 @@ def content_folder_tags(config: dict) -> set[str]:
         return tags
     rows = config.get("_rom_folders") or []
     folders = {str(r.get("folder")) for r in rows if r.get("folder")}
-    pairs: set[tuple[str, str]] = set()
-    for core, systems in (config.get("libretro_systems") or {}).items():
-        pairs |= {(_core_key(core), str(s)) for s in systems or []}
-    for row in rows:
-        choices = [row, *(row.get("board_defaults") or {}).values(), *(row.get("offers") or [])]
-        for choice in choices:
-            if choice.get("core") and choice.get("emulator") in LIBRETRO_RUNNERS:
-                pairs.add((_core_key(choice["core"]), str(row["folder"])))
-    for system in (config.get("systems") or {}).values():
-        if isinstance(system, dict) and system.get("native_id"):
-            pairs |= {(_core_key(c), str(system["native_id"])) for c in system.get("cores") or []}
-    for record in config.get("launchers") or []:
-        if record.get("kind") == "package" and record.get("build") == "libretro":
-            package = str(record.get("emulator", ""))
-            if package.startswith("lr-mess"):
-                # Built with SUBTARGET=mess (lr-mess2016.sh:31): the MESS
-                # drivers alone, none of the arcade sets of the core it
-                # shares a repository with.
-                continue
-            core = _core_key(package.removeprefix("lr-"))
-            pairs |= {(core, str(s)) for s in record.get("systems") or []}
+    boards = content_folder_boards(config)
+    target = config.get(TARGET_KEY)
+    pairs = {pair for pair, on in boards.items() if not target or on is None or target in on}
     from launchers import _roms_place  # noqa: PLC0415
 
     def place(folder: str) -> list:
         return list(_roms_place(config, folder) or ())
 
+    # Without a board, a core a folder offers on some boards only carries
+    # them, for a manifest the installer narrows to one.
     tags |= {
-        CONTENT_FOLDER_PREFIX + json.dumps([c, f, place(f)]) for c, f in pairs if f in folders
+        CONTENT_FOLDER_PREFIX + json.dumps([
+            c, f, place(f),
+            None if target or boards.get((c, f)) is None else sorted(boards[(c, f)]),
+        ])
+        for c, f in pairs if f in folders
     }
     for row in rows:
         for search in row.get("searched_by") or []:
@@ -3529,6 +3618,10 @@ def _rom_search_sets(
     if relation is None:
         return frozenset()
     bios, devices = relation.get("bios") or {}, relation.get("devices") or {}
+    # A clone reads the ROMs it shares with its parent from the parent's
+    # archive: the search path is the set and every ancestor
+    # (src/emu/driver.cpp:31-38, src/emu/romload.cpp:67-84 at mame0289).
+    parents = relation.get("parents") or {}
     sources = relation.get("sources") or {}
     members: set[str] = set(machines)
     kind, keys = ANY_MACHINE_POPULATIONS.get(system, ("all", ())) if any_machine else ("", ())
@@ -3544,6 +3637,7 @@ def _rom_search_sets(
         return kind == "all" or not members.isdisjoint(users)
 
     found = {b for b, users in bios.items() if used(users)}
+    found |= {p for p, users in parents.items() if used(users)}
     found |= {d for d, users in devices.items() if used(users)}
     return frozenset(found | members)
 
@@ -3601,12 +3695,19 @@ def content_layout(
         return {**entry, **where, "path": f"{folder}/{rel}", "layout_path": rel}
 
     if not standalone:
-        folders = sorted({
-            tuple(where) for core, _folder, where in (
-                json.loads(t[len(CONTENT_FOLDER_PREFIX):])
-                for t in tags if t.startswith(CONTENT_FOLDER_PREFIX)
-            ) if core in names and where
-        })
+        # A place keeps the boards its core is offered on there, every board
+        # when one folder offers it everywhere.
+        on_boards: dict[tuple, set[str] | None] = {}
+        for core, _folder, where, on in (
+            json.loads(t[len(CONTENT_FOLDER_PREFIX):])
+            for t in tags if t.startswith(CONTENT_FOLDER_PREFIX)
+        ):
+            if core not in names or not where:
+                continue
+            place = tuple(where)
+            known = on_boards.get(place, set())
+            on_boards[place] = None if on is None or known is None else known | set(on)
+        folders = sorted(on_boards)
         # The folders a passed `-rompath` replaces the build's own search
         # path with stand for these: libretro MAME hands MAME
         # <system>/mame/bios unless the content's command line names
@@ -3633,7 +3734,11 @@ def content_layout(
                 out.append({**plain, "unplaced": OUTSIDE_BIOS_FOLDER})
                 continue
             rel = str(f.get("path") or f.get("name") or "")
-            out.extend(roms_entry(plain, tree, folder, rel) for tree, folder in folders)
+            out.extend(
+                {**roms_entry(plain, tree, folder, rel),
+                 **({"boards": sorted(on)} if (on := on_boards[(tree, folder)]) is not None else {})}
+                for tree, folder in folders
+            )
         return {**view, "files": out} if changed else view
     pools = {str(v).strip("/") for v in (view.get("standalone_pools") or {}).values()}
     if not searches or "" not in pools:
@@ -3649,7 +3754,7 @@ def content_layout(
         if f.get("unplaced") != OUTSIDE_BIOS_FOLDER:
             # A set a launcher record placed stays there: EmuDeck's
             # mame.ini rompath is read when Steam ROM Manager starts MAME
-            # with no `-rompath` (tools/launchers/mame.sh), and ES-DE's
+            # with no -rompath (tools/launchers/mame.sh), and ES-DE's
             # commands read the folders they pass.
             out.append(f)
             held = {(f.get("root") or "", f.get("standalone_path") or "")}

scripts/generate_games_index.py

@@ -543,10 +543,10 @@ def build_systems(
 def build_arcade(directory: Path) -> dict[str, dict]:
     if not directory.is_dir():
         return {}
-    # The driver sources place a search path's sets at build time
-    # (common.rom_search_sets): the installer reads none of them.
+    # The driver sources and the clone parents place a search path's sets at
+    # build time (common.rom_search_sets): the installer reads none of them.
     return {
-        path.stem: {k: v for k, v in _load_json(path).items() if k != "sources"}
+        path.stem: {k: v for k, v in _load_json(path).items() if k not in ("sources", "parents")}
         for path in sorted(directory.glob("*.json"))
     }
 

scripts/generate_pack.py

@@ -39,6 +39,10 @@ sys.path.insert(0, os.path.dirname(__file__))
 from alternatives import core_file_spares, settle_sets, slot_set_keys
 from common import resolve_platform_cores as _platform_cores
 from common import (
+    ROMS_ROOT,
+    TARGET_KEY,
+    _core_key,
+    content_folder_boards,
     OUTER_ROOTS,
     outer_layout,
     installed_data_refs,
@@ -895,6 +899,8 @@ def generate_pack(
     Returns the path to the generated ZIP, or None on failure.
     """
     config = load_platform_config(platform_name, platforms_dir)
+    if target_name:
+        config = {**config, TARGET_KEY: target_name}
     if zip_contents is None:
         zip_contents = {}
     if emu_profiles is None:
@@ -4861,6 +4867,8 @@ def generate_manifest(
     config = load_platform_config(platform_name, platforms_dir)
     if fold_case is not None:
         config = {**config, "case_insensitive_fs": fold_case}
+    if target_name:
+        config = {**config, TARGET_KEY: target_name}
     if zip_contents is None:
         zip_contents = {}
     if emu_profiles is None:
@@ -5334,6 +5342,10 @@ def generate_manifest(
         **({"downloaded_cores": downloaded} if (downloaded := _downloaded_cores(
             config, emu_profiles, core_names, platforms_dir,
         )) else {}),
+        **({"offered_boards": offered} if (offered := _offered_boards(
+            config, emu_profiles, core_names,
+            [*manifest_files, *omitted_by_destination.values()],
+        )) else {}),
         "system_cores": system_cores,
         # The names the platform itself gives its systems (Batocera's psx),
         # which --system accepts beside the ids.
@@ -5488,6 +5500,65 @@ def _manifest_data_directories(
     return trees
 
 
+def _offered_boards(
+    config: dict,
+    profiles: dict[str, dict],
+    core_names: dict[str, list[str]],
+    entries: list[dict],
+) -> dict[str, list[dict]]:
+    """The roms folders each profile is offered for on some boards only,
+    each with those boards, aliases included, where the manifest places a
+    file of that profile.
+
+    ROCKNIX writes uae4arm for roms/amiga under the devices it builds it for
+    (emulators/package.mk:188), and a board's core list names uae4arm when
+    another folder offers it there. The pack builder reads the board from
+    the config (TARGET_KEY) and `install.py --target` reads its answer here.
+    A manifest built for one board needs none.
+    """
+    if config.get(TARGET_KEY):
+        return {}
+    pairs = content_folder_boards(config)
+    if all(on is None for on in pairs.values()):
+        return {}
+    from launchers import _roms_place  # noqa: PLC0415
+
+    by_key: dict[str, set[str]] = {}
+    for name, claimants in upstream_profile_index(profiles).items():
+        by_key.setdefault(_core_key(name), set()).update(claimants)
+    free = {
+        (profile, folder) for (core, folder), on in pairs.items() if on is None
+        for profile in by_key.get(core, ())
+    }
+    found: dict[tuple[str, str, str], set[str]] = {}
+    for (core, folder), on in pairs.items():
+        where = _roms_place(config, folder) if on is not None else None
+        if where is None:
+            continue
+        for profile in by_key.get(core, ()):
+            if profile in core_names and (profile, folder) not in free:
+                found.setdefault((profile, where[0], where[1]), set()).update(on)
+    # An entry below the roms root names it, one in the install tree none.
+    trees = {ROMS_ROOT: ROMS_ROOT, None: "install"}
+    placed = {
+        (profile, trees[entry.get("root")], str(entry.get("dest", "")).rpartition("/")[0])
+        for entry in entries if entry.get("root") in trees
+        for copy in [entry, *entry.get("alternatives", [])]
+        for profile in copy.get("cores") or []
+    }
+    result: dict[str, list[dict]] = {}
+    for (profile, tree, folder), boards in sorted(found.items()):
+        if not any(
+            p == profile and t == tree and (d == folder or d.startswith(f"{folder}/"))
+            for p, t, d in placed
+        ):
+            continue
+        result.setdefault(profile, []).append(
+            {"root": tree, "folder": folder, "boards": sorted(boards)}
+        )
+    return result
+
+
 def _downloaded_cores(
     config: dict,
     profiles: dict[str, dict],

scripts/profile_sync.py

@@ -3682,10 +3682,11 @@ def bump_commit(
     if pending_recale(report, accept_changed, text):
         return False
     document = yaml.safe_load(text)
-    if document.get("built_by"):
+    if document.get("built_by") and not document.get("pin_follows"):
         # The pin is the commit a platform builds (recalbox gsplus.mk, the
         # mame0258 its package fetches), not a point on a moving branch:
-        # HEAD is a build nobody ships under this name.
+        # HEAD is a build nobody ships under this name. A build of the
+        # branch tip (RetroPie's advmame) is HEAD, and follows it.
         print(
             f"{path.stem}: pinned to the build {', '.join(document['built_by'])} "
             "ships, advanced by hand",
@@ -3744,6 +3745,8 @@ def bump_commit(
     expected = dict(document)
     expected["source_commit"] = report.head
     new_text = _set_pin(text, document, "source_commit", report.head, "profiled_date")
+    if document.get("pin_follows"):
+        new_text, expected = _follow_config_reads(new_text, expected, document, report.head)
     if upstream_moves:
         expected["upstream_commit"] = report.upstream_head
         new_text = _set_pin(
@@ -3800,6 +3803,28 @@ def cites_primary(report: ProfileReport) -> bool:
     return not parts or any(part.repo is None for part in parts)
 
 
+def _follow_config_reads(
+    text: str, expected: dict, document: dict, head: str
+) -> tuple[str, dict]:
+    """A config_reads entry read at the pin moves with a pin that follows the
+    branch tip: RetroPie installs advmame from the tip, so the build its
+    advmame.rc is read at is the pin itself."""
+    pin = str(document.get("source_commit") or "")
+    reads = document.get("config_reads") or []
+    if not pin or not any(isinstance(r, dict) and r.get("revision") == pin for r in reads):
+        return text, expected
+    text = re.sub(
+        r'^(\s*revision:\s*)(["\']?)' + re.escape(pin) + r"\2\s*$",
+        lambda m: f"{m.group(1)}{m.group(2)}{head}{m.group(2)}",
+        text, flags=re.M,
+    )
+    expected = {**expected, "config_reads": [
+        {**r, "revision": head} if isinstance(r, dict) and r.get("revision") == pin else r
+        for r in reads
+    ]}
+    return text, expected
+
+
 def _set_pin(text: str, document: dict, field: str, value: str, after: str) -> str:
     """Write *field* as *value*, replacing its line or adding one after *after*."""
     current = document.get(field)

scripts/scraper/arcade_parent_importer.py

@@ -205,6 +205,7 @@ def build_index(
         if n not in bios_names and not e.isdevice and e.runnable and (e.has_roms or e.disks)
     }
     bios: dict[str, set[str]] = {}
+    parents: dict[str, set[str]] = {}
     devices: dict[str, set[str]] = {}
     missing: dict[str, str] = {}
     for name, entry in games.items():
@@ -218,6 +219,12 @@ def build_index(
                 break
             if current in bios_names and parent.has_roms:
                 bios.setdefault(archive(current, current), set()).add(name)
+            elif current not in bios_names and parent.has_roms:
+                # A clone's ROM shared with its parent is looked up in the
+                # parent's archive too: MAME's search path is the set and
+                # every ancestor (src/emu/driver.cpp:31-38,
+                # src/emu/romload.cpp:67-84 at mame0289).
+                parents.setdefault(archive(current, current), set()).add(name)
             current = parent.romof
         for device in set(entry.devices) & device_roms:
             devices.setdefault(device, set()).add(name)
@@ -231,6 +238,7 @@ def build_index(
             sources.setdefault(entry.source, set()).add(archive(name, name))
     return {
         "bios": {k: sorted(v) for k, v in sorted(bios.items())},
+        "parents": {k: sorted(v) for k, v in sorted(parents.items())},
         "devices": {k: sorted(v) for k, v in sorted(devices.items())},
         "chd": sorted(n for n, e in games.items() if e.disks),
         "missing_parents": dict(sorted(missing.items())),

scripts/scraper/retropie_launchers.py

@@ -473,15 +473,26 @@ class Scripts:
         return results
 
 
-def _search_list_in_roms(event: "Event") -> bool:
+# The character each program splits a search list on, read in its code.
+# AdvanceMAME strtoks dir_rom on file_dir_separator(), ':' on Linux, in all
+# three builds RetroPie ships (advance/linux/file.c:138-141 and
+# advance/osd/fileio.c:696-699 at 96237a5, 685-688 at advancemame-1.4,
+# 640-643 in advancemame-0.94.0.tar.gz). MAME's path_iterator splits on ';'
+# whatever the system (src/emu/fileio.cpp:56, 64, 112 at mame0289).
+LIST_SEPARATORS = {"advmame": ":", "advmame-0.94": ":", "advmame-1.4": ":"}
+DEFAULT_LIST_SEPARATOR = ";"
+
+
+def _search_list_in_roms(event: "Event", separator: str = DEFAULT_LIST_SEPARATOR) -> bool:
     """An iniSet whose value is a list of folders below the roms root.
 
     One folder alone is where a program writes (cfg_directory, nvram_directory
-    and the others MAME's mame.ini sets): a list is where it searches.
+    and the others MAME's mame.ini sets): a list is where it searches, split
+    on the separator the program reads it with.
     """
     if len(event.words) < 3 or len(event.words[2]) != 1:
         return False
-    parts = [p for p in event.words[2][0].split(";") if p]
+    parts = [p for p in event.words[2][0].split(separator) if p]
     return len(parts) > 1 and all(
         p.startswith(ROMDIR + "/") or p.startswith("${romdir}/") for p in parts
     )
@@ -567,7 +578,9 @@ class PackageReader:
         if command == "iniSet" and not _mentions_bios(event.raw):
             # A search path below the roms root: MAME reads its samples from
             # samplepath $romdir/mame/samples;$romdir/arcade/samples.
-            if _search_list_in_roms(event):
+            if _search_list_in_roms(
+                event, LIST_SEPARATORS.get(self.package.id, DEFAULT_LIST_SEPARATOR)
+            ):
                 self._ini_set(event)
             return
         if not _mentions_bios(event.raw):

scripts/verify.py

@@ -46,6 +46,8 @@ from alternatives import (
     slot_set_keys,
 )
 from common import (
+    TARGET_KEY,
+    canonical_target_name,
     OUTER_ROOTS,
     _foreign_tree,
     _owner_tokens,
@@ -3302,6 +3304,11 @@ def _main(held: contextlib.ExitStack) -> None:
     for group_platforms, representative in groups:
         config = load_platform_config(representative, args.platforms_dir)
         tc = target_cores_cache.get(representative) if args.target else None
+        if tc is not None:
+            # A core a roms folder offers on other boards only is not placed
+            # there, as generate_pack builds the board's pack.
+            config = {**config, TARGET_KEY: canonical_target_name(
+                [representative], args.target, args.platforms_dir)}
         result = verify_platform(
             config,
             db,