chore: merge the integration into site-text

scripts/common.py

@@ -2760,27 +2760,49 @@ def expand_directory_entries(
 
     if not any(entry.get("type") == "directory" for entry in files):
         return files
+    # A file the profile names itself is left to that entry, in the same
+    # build and below the same root: MAME's libretro floppy samples share
+    # their path with the standalone tree RetroPie reads below roms/.
+    other_build = "libretro" if standalone else "standalone"
     named = {
-        destination(entry) for entry in files if entry.get("type") != "directory"
+        (entry.get("root") or "", destination(entry)) for entry in files
+        if entry.get("type") != "directory" and entry.get("mode") != other_build
     }
     expanded: list[dict] = []
     for entry in files:
         if entry.get("type") != "directory":
             expanded.append(entry)
             continue
-        members = directory_members(db, destination(entry))
+        target = destination(entry)
+        # A directory a platform moved (platform_layouts) is the profile's
+        # tree at the platform's place: the collection holds it under the
+        # layout path, and the platform's folder proves nothing (RetroDECK
+        # hands MAME samplepath $bios_path/mame-sa/samples).
+        layout = str(entry.get("layout_path") or "") if standalone else ""
+        members = directory_members(db, layout) if layout else {}
+        if not members:
+            # A layout tail two trees answer (MAME's hash/) leaves the
+            # platform's own tail.
+            layout = ""
+            members = directory_members(db, target)
+        if layout and layout.strip("/") != target.strip("/"):
+            source, place = layout.strip("/") + "/", target.strip("/") + "/"
+            members = {
+                place + dest[len(source):]: sha1
+                for dest, sha1 in members.items() if dest.startswith(source)
+            }
         if not members:
             expanded.append(entry)
             continue
         inherited = {
             key: value
             for key, value in entry.items()
-            if key not in ("type", "standalone_path")
+            if key not in ("type", "standalone_path", "layout_path")
         }
         expanded.extend(
             {**inherited, "name": dest, "path": dest, "sha1": sha1}
             for dest, sha1 in members.items()
-            if dest not in named
+            if (entry.get("root") or "", dest) not in named
         )
     return expanded
 
@@ -2905,6 +2927,32 @@ def own_build_tags(config: dict, standalone: set[str]) -> set[str]:
     }
 
 
+def twin_build_names(config: dict) -> set[str]:
+    """The libretro core names of emulators a platform also runs standalone
+    under another name.
+
+    RetroBat's es_systems.cfg offers MAME as the libretro core mame and as
+    the standalone mame64, two names of one profile: the platform runs it
+    both ways, and each build reads its own layout.
+    """
+    standalone = {str(c) for c in config.get("standalone_cores") or []}
+    libretro = {str(c) for c in config.get("cores") or []} - standalone if isinstance(
+        config.get("cores"), list
+    ) else set()
+    if not standalone or not libretro:
+        return set()
+    profiles = load_emulator_profiles(str(Path(__file__).resolve().parent.parent / "emulators"))
+    found: set[str] = set()
+    for key, profile in profiles.items():
+        kind = str(profile.get("type", ""))
+        if "libretro" not in kind or "standalone" not in kind:
+            continue
+        names = {key, *(str(c) for c in profile.get("cores") or [])}
+        if names & standalone:
+            found |= names & libretro
+    return found
+
+
 def standalone_set(config: dict) -> set[str]:
     """The emulators a platform launches standalone, with its dual-mode ones.
 
@@ -2924,6 +2972,7 @@ def standalone_set(config: dict) -> set[str]:
         & set(map(str, config.get("standalone_cores") or []))
     }
     names |= own_build_tags(config, names)
+    names |= {DUAL_MODE_PREFIX + n for n in twin_build_names(config)}
     # The system directory each core is handed travels the same way, so
     # platform_layouts places a core's files below it for every reader.
     names |= {
@@ -3157,6 +3206,29 @@ def standalone_bindings_for(
     )
 
 
+def _placed_entry(entry: dict, profile: dict, placed: tuple[str, str]) -> dict:
+    """A standalone entry moved to the (tree, path) a platform reads it from."""
+    tree, path = placed
+    moved = {**entry, "standalone_path": path,
+             "layout_path": standalone_layout_path(entry, profile)}
+    if entry.get("load_from"):
+        # The build's own root no longer applies: RetroDECK hands
+        # Cemu's MLC, read from its data directory, to bios/cemu.
+        load_from = entry["load_from"]
+        moved["load_from"] = {
+            **(load_from if isinstance(load_from, dict) else {"libretro": load_from}),
+            "standalone": "system_dir",
+        }
+    if tree == "install":
+        # Below the roms root inside the install tree (AmberELEC's
+        # /storage/roms): a plain dest the platform prefix skips.
+        return {**moved, "unprefixed": True}
+    # A file the launcher runs from the roms directory (ROCKNIX's
+    # roms/pico-8) or an emulator folder (RetroBat's emulators/pico8)
+    # is relative to it, and says so.
+    return {**moved, "root": tree} if tree else moved
+
+
 def platform_layouts(
     emu_name: str, profile: dict, standalone_cores: set[str]
 ) -> list[tuple[dict, bool]]:
@@ -3191,7 +3263,14 @@ def platform_layouts(
             for system in own:
                 family.setdefault(system, set()).update(entry_launch_systems(f, index) - own)
 
-    def bindings_of(entry: dict) -> dict[str, tuple[str, str]]:
+    # The systems the launcher records name: a launch for one of them reads
+    # the folder its record gives, every other launch the folder the records
+    # naming none give.
+    named_systems = frozenset().union(*(
+        h.systems for h in _handed(standalone_cores) if h.emulator in names
+    )) if per_system else frozenset()
+
+    def launch_systems(entry: dict) -> frozenset[str]:
         systems: frozenset[str] = frozenset()
         if per_system:
             own = entry_launch_systems(entry, {})
@@ -3205,12 +3284,53 @@ def platform_layouts(
                 systems = systems.union(*(
                     index.get(MACHINE_KEY + _norm_system_id(s), set()) for s in own
                 ))
+        return systems
+
+    def bindings_for(systems: frozenset[str]) -> dict[str, tuple[str, str]]:
         if systems not in by_systems:
             by_systems[systems] = standalone_bindings_for(
                 emu_name, profile, standalone_cores, systems
             )
         return by_systems[systems]
 
+    def bindings_of(entry: dict) -> dict[str, tuple[str, str]]:
+        return bindings_for(launch_systems(entry))
+
+    # The layout folders a search setting stands for (standalone_pools):
+    # bound, the launcher passes the setting and the default is not read.
+    pool_folders = {
+        str(v).strip("/") for v in (profile.get("standalone_pools") or {}).values()
+    } - {""}
+
+    def passed_setting(entry: dict) -> bool:
+        layout = str(standalone_layout_path(entry, profile) or "")
+        bindings = bindings_of(entry)
+        return any(
+            folder in bindings and (layout == folder or layout.startswith(folder + "/"))
+            for folder in pool_folders
+        )
+
+    def destinations(entry: dict) -> list[tuple[str, str] | None]:
+        """Each place the entry's launches read it from, in launch order.
+
+        An entry of a system a record names is read where that record puts
+        it. One also launched for systems no record names is read, for those
+        launches, where the records naming none put it: EmuELEC starts
+        AdvanceMAME with dir_rom roms/arcade for arcade and roms/mame for
+        mame and cps1-3 (advmame.sh:26-30), and an arcade set is read in both.
+        """
+        systems = launch_systems(entry)
+        found = [standalone_destination(entry, profile, bindings_for(systems))]
+        # A system the platform's list or machines put the entry under
+        # stands for the entry's own ids: XRoar's coco3 is launched as coco.
+        own = entry_launch_systems(entry, {})
+        launched = (systems - own) | (own & named_systems) if systems != own else systems
+        if launched & named_systems and launched - named_systems:
+            generic = standalone_destination(entry, profile, bindings_for(frozenset()))
+            if generic not in found:
+                found.append(generic)
+        return found
+
     layout_entries = [
         f for f in profile.get("files") or []
         if isinstance(f, dict) and standalone_layout_path(f, profile) is not None
@@ -3228,30 +3348,18 @@ def platform_layouts(
             if not isinstance(f, dict) or standalone_layout_path(f, profile) is None:
                 files.append(f)
                 continue
-            placed = standalone_destination(f, profile, bindings_of(f))
-            if placed is None or placed[0] == "outside":
+            places = [p for p in destinations(f) if p is not None and p[0] != "outside"]
+            if not places:
                 files.append({**f, "unplaced": OUTSIDE_BIOS_FOLDER})
                 continue
-            tree, path = placed
-            moved = {**f, "standalone_path": path,
-                     "layout_path": standalone_layout_path(f, profile)}
-            if f.get("load_from"):
-                # The build's own root no longer applies: RetroDECK hands
-                # Cemu's MLC, read from its data directory, to bios/cemu.
-                load_from = f["load_from"]
-                moved["load_from"] = {
-                    **(load_from if isinstance(load_from, dict) else {"libretro": load_from}),
-                    "standalone": "system_dir",
-                }
-            if tree == "install":
-                # Below the roms root inside the install tree (AmberELEC's
-                # /storage/roms): a plain dest the platform prefix skips.
-                files.append({**moved, "unprefixed": True})
-                continue
-            # A file the launcher runs from the roms directory (ROCKNIX's
-            # roms/pico-8) or an emulator folder (RetroBat's emulators/pico8)
-            # is relative to it, and says so.
-            files.append({**moved, "root": tree} if tree else moved)
+            moved = f
+            if f.get("bundled") and passed_setting(f):
+                # The copy installed with the emulator sits where the setting
+                # pointed before the launcher moved it: RetroBat passes MAME
+                # -hash bios/mame/hash (Mame64.Generator.cs:172, 177-178), and
+                # the hash/ beside mame.exe is no longer read.
+                moved = {k: v for k, v in f.items() if k != "bundled"}
+            files.extend(_placed_entry(moved, profile, placed) for placed in places)
         standalone = {**profile, "files": files}
     dual = {c.removeprefix(DUAL_MODE_PREFIX) for c in standalone_cores
             if c.startswith(DUAL_MODE_PREFIX)}

scripts/generate_event_schema.py

@@ -37,10 +37,11 @@ KEY_TYPES: dict[str, dict] = {
     **dict.fromkeys((
         "installer", "os", "os_name", "stage", "reason", "platform", "message",
         "folder", "dest", "system", "name", "bios_dir", "path", "member",
-        "list_version", "installed_version", "board", "core",
+        "list_version", "installed_version", "board", "core", "date",
     ), _TEXT),
     "mode": {"enum": ["install", "check", "dry-run", "list"]},
     "state": {"enum": ["present", "wrong_hash", "missing"]},
+    "status": {"enum": ["applied", "written", "unreadable", "not_written"]},
     **dict.fromkeys((
         "files", "files_done", "files_total", "bytes", "bytes_done", "bytes_total",
         "bytes_local", "bytes_network", "size", "present", "missing", "wrong_hash",
@@ -48,7 +49,7 @@ KEY_TYPES: dict[str, dict] = {
         "pending", "code", "games", "recognised", "not_games",
     ), _COUNT),
     **dict.fromkeys(("truncated", "ok", "interrupted", "required_only", "applied", "info"), _FLAG),
-    **dict.fromkeys(("systems", "other_drives", "dests", "written"), _NAMES),
+    **dict.fromkeys(("systems", "other_drives", "dests", "written", "choice"), _NAMES),
     **dict.fromkeys(("platforms", "failed", "skipped", "left_out", "kept", "found",
                      "fallback"), _RECORDS),
 }

scripts/launchers.py

@@ -314,8 +314,9 @@ BIOS_ROOT_VARIABLES = ("$biosdir", "$home/RetroPie/BIOS", "${biosPath}", "$bios_
 
 # The name a platform's launch code gives its roms root: RetroDECK's
 # component scripts write $roms_path (retrodeck_launchers), the setting whose
-# default is the roms root (config/retrodeck/retrodeck.json:5).
-ROMS_ROOT_VARIABLES = ("$roms_path", "${roms_path}")
+# default is the roms root (config/retrodeck/retrodeck.json:5), RetroPie's
+# scriptmodules $romdir (retropie_packages.sh:47-49).
+ROMS_ROOT_VARIABLES = ("$roms_path", "${roms_path}", "$romdir", "${romdir}")
 
 # The kinds that hand an emulator a whole directory. A scans_dir record sets
 # the directory the emulator picks its file from: ROCKNIX writes ARMSX2's

scripts/packextras.py

@@ -1545,6 +1545,11 @@ def _collect_emulator_extras(
                     extra["system_dir"] = f["system_dir"]
                 if f.get("unprefixed"):
                     extra["unprefixed"] = True
+                # A copy placed outside the BIOS tree resolves by its layout
+                # path alone, as the first pass does: EmuELEC's
+                # roms/mame/stvbios.zip took bios/Arcade/MAME's set by that tail.
+                if f.get("layout_path"):
+                    extra["layout_path"] = f["layout_path"]
                 extra["claim_order"] = (2, 0, emu_name)
                 if _take(extra, full_dest, seen_dests, taken):
                     extras.append(extra)

scripts/release_notes.py

@@ -238,9 +238,9 @@ def render(record: dict, platforms_dir: str, whats_new: str,
     folders = _extract_folders(record, platforms_dir)
     tag = str(record.get("tag", ""))
     lines = [
-        "### Get your BIOS",
+        "### Get the BIOS files",
         "",
-        "Paste this in a terminal. It finds your setup and installs the missing BIOS files.",
+        "Paste this in a terminal. It finds the setup and installs the missing BIOS files.",
         "",
         "**Linux, macOS, Steam Deck**",
         "```sh",
@@ -252,11 +252,11 @@ def render(record: dict, platforms_dir: str, whats_new: str,
         install_command("powershell"),
         "```",
         "",
-        f"**On a handheld**: [get the app for your system]({SITE}{HANDHELD_PAGE}).",
+        f"**On a handheld**: [get the app for the system]({SITE}{HANDHELD_PAGE}).",
         "",
         "### Full packs",
         "",
-        "Large downloads, for a device without internet. Extract into your BIOS folder.",
+        "Large downloads, for a device without internet. Extract into the BIOS folder.",
         "",
     ]
     if any(len(pack["assets"]) > 1 for pack in packs.values()):

scripts/scraper/_hash_merge.py

@@ -578,8 +578,10 @@ def _append_new_entries(text: str, files: list[dict], original: str) -> str:
         return text
 
     lines = []
+    profile_type = str(existing_data.get("type") or "")
     for fe in new_entries:
         lines.append(f"\n  - name: {_yaml_scalar(fe['name'])}")
+        lines.extend(set_path_lines(profile_type, fe["name"]))
         lines.append(f"    required: {str(fe['required']).lower()}")
         lines.append("    category: bios_zip")
         if fe.get("source_ref"):
@@ -715,10 +717,27 @@ def _rewrite_systems(section: str, systems: dict[str, list[str]]) -> str:
     return "\n".join(out)
 
 
-def _format_device_entry(name: str, device: dict, role: str) -> str:
+def set_path_lines(profile_type: str, name: str) -> list[str]:
+    """Where each build of a MAME profile reads a set it lists.
+
+    The libretro build searches the content folder, then <system>/mame/bios
+    and <system>/mame/roms (libretro-internal/retro_init.cpp,
+    Set_Path_Option), never the system folder itself. The standalone build
+    reads the top of its rompath.
+    """
+    if "libretro" not in profile_type:
+        return []
+    lines = [f'    path: "mame/bios/{name}"']
+    if "standalone" in profile_type:
+        lines.append(f'    standalone_path: "{name}"')
+    return lines
+
+
+def _format_device_entry(name: str, device: dict, role: str, profile_type: str = "") -> str:
     """One generated device entry, in the field order of the written ones."""
     lines = [
         f"  - name: {_zip_scalar(name)}",
+        *set_path_lines(profile_type, name),
         "    required: false",
         "    category: bios_zip",
         *_system_lines(device.get("systems") or []),
@@ -779,7 +798,9 @@ def merge_mame_devices(
         name = f"{set_name}.zip"
         existing = entries.get(name)
         if not existing:
-            new_blocks[name] = _format_device_entry(name, device, role)
+            new_blocks[name] = _format_device_entry(
+                name, device, role, str(profile.get("type") or "")
+            )
             report["added"].append(set_name)
             continue
         if any(e.get("category") != "bios_zip" for e in existing):

scripts/scraper/batocera_launchers.py

@@ -1119,10 +1119,32 @@ def patched_paths(read: Callable[[str], str], patches: tuple[str, ...] = PATH_PA
     return launchers
 
 
+# Generators whose command line hands the emulator /userdata/bios folders as
+# search paths: MAME's -rompath and -samplepath (mame/mameGenerator.py).
+SEARCH_PATH_GENERATORS = ("mame/mameGenerator.py",)
+
+
+def search_paths(cg: Configgen) -> list[Launcher]:
+    """Every /userdata/bios path a generator's command line names, read as
+    KNULLI's generic reader reads its fork of the same generator."""
+    try:
+        from .knulli_launchers import GenericSites, bios_sites  # noqa: PLC0415
+    except ImportError:
+        from knulli_launchers import GenericSites, bios_sites  # type: ignore[no-redef]  # noqa: PLC0415
+    # A boot disk image the command line names (the Macintosh's
+    # /userdata/bios/<boot_disk>.img) is a file, no search path.
+    return [
+        launcher
+        for path in SEARCH_PATH_GENERATORS
+        for launcher in bios_sites(cg, GenericSites(path, "passes_path"))
+        if re.match(r"command line -\w", launcher.passes_to or "")
+    ]
+
+
 EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = (
     duckstation, duckstation_legacy, hatari, xemu, melonds, mupen64plus,
     x16emu, gsplus, lexaloffle, environment_directories, config_directories,
-    core_file_options, working_directories,
+    core_file_options, working_directories, search_paths,
 )
 
 

scripts/scraper/mame_hash_scraper.py

@@ -453,6 +453,10 @@ def release_profile(
         name = f"{set_name}.zip"
         entry: dict[str, Any] = {
             "name": name,
+            # The libretro build searches the content folder, then
+            # <system>/mame/bios and <system>/mame/roms (retro_init.cpp,
+            # Set_Path_Option): the system folder itself is not read.
+            "path": f"mame/bios/{name}",
             "required": settled_required(known.get(name, {}), data),
             "category": "bios_zip",
         }

scripts/scraper/retropie_launchers.py

@@ -32,6 +32,9 @@ SECTIONS = ("emulators", "libretrocores", "ports")
 PACKAGES_SCRIPT = "retropie_packages.sh"
 
 BIOSDIR = "$biosdir"
+# The roms root (retropie_packages.sh:47-49): MAME's mame.ini lists rom,
+# sample and artwork folders below it (scriptmodules/emulators/mame.sh:133-136).
+ROMDIR = "$romdir"
 # The package whose config every lr- core runs under.
 LIBRETRO_FRONTEND = "retroarch"
 # The launch scripts copied next to an emulator spell the directory out.
@@ -464,6 +467,20 @@ class Scripts:
         return results
 
 
+def _search_list_in_roms(event: "Event") -> bool:
+    """An iniSet whose value is a list of folders below the roms root.
+
+    One folder alone is where a program writes (cfg_directory, nvram_directory
+    and the others MAME's mame.ini sets): a list is where it searches.
+    """
+    if len(event.words) < 3 or len(event.words[2]) != 1:
+        return False
+    parts = [p for p in event.words[2][0].split(";") if p]
+    return len(parts) > 1 and all(
+        p.startswith(ROMDIR + "/") or p.startswith("${romdir}/") for p in parts
+    )
+
+
 def _mentions_bios(text: str) -> bool:
     return BIOSDIR in text or "${biosdir}" in text or BIOS_SPELLED in text
 
@@ -541,6 +558,12 @@ class PackageReader:
         if command == "addEmulator":
             self._add_emulator(event)
             return
+        if command == "iniSet" and not _mentions_bios(event.raw):
+            # A search path below the roms root: MAME reads its samples from
+            # samplepath $romdir/mame/samples;$romdir/arcade/samples.
+            if _search_list_in_roms(event):
+                self._ini_set(event)
+            return
         if not _mentions_bios(event.raw):
             return
         if command in HOUSEKEEPING:

scripts/validate_schemas.py

@@ -87,6 +87,7 @@ def _validate_yaml_directory(
             out.extend(_systems_outside_profile(path, data))
             out.extend(_doubled_paths(path))
             out.extend(_group_requirement_split(path, data))
+            out.extend(_data_only_source_refs(path, data))
     return out
 
 
@@ -155,6 +156,65 @@ def _group_requirement_split(path: Path, document: object) -> list[str]:
     return out
 
 
+_CODE_SUFFIXES = frozenset(
+    "c cc cpp cxx c++ h hh hpp hxx inc inl ipp m mm mjs cjs js jsx ts tsx py cs java kt "
+    "go rs lua pas pp asm s sh bash zsh ps1 bat cmd vbs swift d zig nim qml qc rb pl php "
+    "scala ml hs dart vue ino z80 smali".split()
+)
+_CODE_FILE = re.compile(r"[\w+.-]\.([A-Za-z0-9+]+)\b")
+_BINARY_EVIDENCE = re.compile(
+    r"\b0x[0-9a-fA-F]+\b|\bFUN_[0-9a-fA-F]+\b|\.(?:so|dll|exe|dylib|dex|apk|jar|elf|nro|xex)\b",
+    re.IGNORECASE,
+)
+
+# Profiles whose files cite no published code. Each reason says why the
+# check cannot apply yet.
+DATA_ONLY_REF_EXCEPTIONS: dict[str, str] = {
+    "ace-dl": "closed source: the refs describe what the shipped binary opens",
+    "cpcemu": "closed source: the refs describe what the shipped binary opens",
+    "drastic": "closed source: the shader format files are read by the shipped binary",
+    "lexaloffle": "closed source: PICO-8 reads pico8_gpio, the refs are its readme and the KNULLI install script",
+    "mfme": "closed source: the refs are the release notes of the shipped binary",
+}
+
+
+def _ref_cites_code(ref: str) -> bool:
+    """A source_ref part names a source file, or a binary with an address."""
+    from sourceref import split_source_ref
+
+    if _BINARY_EVIDENCE.search(ref):
+        return True
+    for part in split_source_ref(ref):
+        for suffix in _CODE_FILE.findall(part.raw):
+            if suffix.lower() in _CODE_SUFFIXES:
+                return True
+    return False
+
+
+def _data_only_source_refs(path: Path, document: object) -> list[str]:
+    """A files[] source_ref that cites only data or build files.
+
+    A data file is evidence that the emulator reads a file only once the
+    code that ships and parses it is cited too: iwadinfo.txt names the
+    IWADs, d_iwad.cpp is what reads it. 655 entries in 27 profiles cited
+    the table alone, and six of them named a file no code opens.
+    """
+    if not isinstance(document, dict) or path.stem in DATA_ONLY_REF_EXCEPTIONS:
+        return []
+    out: list[str] = []
+    for index, entry in enumerate(document.get("files") or []):
+        if not isinstance(entry, dict) or not entry.get("source_ref"):
+            continue
+        refs = entry["source_ref"]
+        for ref in refs.values() if isinstance(refs, dict) else [refs]:
+            if ref and not _ref_cites_code(str(ref)):
+                out.append(
+                    f"{path.relative_to(ROOT)}: files[{index}] ({entry.get('name')}) "
+                    f"source_ref cites no code that reads the file: {ref}"
+                )
+    return out
+
+
 def _checks_without_values(path: Path, document: object) -> list[str]:
     """A check an entry declares with nothing to compare against.