diff --git a/emulators/flycast.yml b/emulators/flycast.yml index c25b9135..f1d50a18 100644 --- a/emulators/flycast.yml +++ b/emulators/flycast.yml @@ -18,6 +18,12 @@ systems: - sega-naomi2 - sega-atomiswave - sega-systemsp +standalone_directories: + - "Dreamcast.BiosPath" + - "find_user_data_dir" + - "$XDG_DATA_HOME/flycast" + - "$HOME/.local/share/flycast" + - "FLYCAST_BIOS_PATH" notes: | The libretro core is built from flyinghead/flycast, the same tree as the @@ -36,6 +42,22 @@ notes: | (shell/libretro/libretro.cpp:2249-2250). All files below are relative to system_dir/dc/. + The standalone emulator looks for each file in the BIOS paths the user + configures, then in its data directory, then in the system data + directories, then in the content directories, and writes the flash back to + the data directory (core/oslib/oslib.cpp:118-196, + core/stdclass.cpp:95-125). The data directory is $XDG_DATA_HOME/flycast/, + else ~/.local/share/flycast/, on Linux (core/linux-dist/main.cpp:74-99), and + the data/ folder of the configuration directory on Windows, Android and + macOS (core/windows/winmain.cpp:73-78, core/windows/winmain.cpp:86-89, + shell/android-studio/flycast/src/main/jni/src/Android.cpp:149-159, + shell/apple/emulator-osx/emulator-osx/osx-main.mm:97-101). + standalone_path is relative to the data directory. The BIOS paths are the + Dreamcast.BiosPath setting (core/cfg/option.cpp:144), the system data + directories take $FLYCAST_BIOS_PATH (core/linux-dist/main.cpp:198-202), and + find_user_data_dir() returns the data directory + (core/linux-dist/main.cpp:74, core/linux-dist/main.cpp:247). + Only the Dreamcast reads flat files. The prefix is dc_ for Dreamcast (core/hw/flashrom/nvmem.cpp:35-50), and the boot ROM is searched as {prefix}boot.bin then {prefix}bios.bin (core/hw/flashrom/nvmem.cpp:304). @@ -65,6 +87,7 @@ files: # ------------------------------------------------------- - name: "dc_boot.bin" path: "dc/dc_boot.bin" + standalone_path: "dc_boot.bin" aliases: ["dc_bios.bin"] system: sega-dreamcast required: false @@ -75,6 +98,7 @@ files: - name: "dc_nvmem.bin" path: "dc/dc_nvmem.bin" + standalone_path: "dc_nvmem.bin" system: sega-dreamcast required: false hle_fallback: true @@ -87,6 +111,7 @@ files: # ------------------------------------------------------- - name: "naomi.zip" path: "dc/naomi.zip" + standalone_path: "naomi.zip" system: sega-naomi required: true category: bios_zip @@ -132,6 +157,7 @@ files: - name: "airlbios.zip" path: "dc/airlbios.zip" + standalone_path: "airlbios.zip" system: sega-naomi required: false category: bios_zip @@ -149,6 +175,7 @@ files: - name: "f355bios.zip" path: "dc/f355bios.zip" + standalone_path: "f355bios.zip" system: sega-naomi required: false category: bios_zip @@ -170,6 +197,7 @@ files: - name: "f355dlx.zip" path: "dc/f355dlx.zip" + standalone_path: "f355dlx.zip" system: sega-naomi required: false category: bios_zip @@ -191,6 +219,7 @@ files: - name: "hod2bios.zip" path: "dc/hod2bios.zip" + standalone_path: "hod2bios.zip" system: sega-naomi required: false category: bios_zip @@ -215,6 +244,7 @@ files: # ------------------------------------------------------- - name: "naomi2.zip" path: "dc/naomi2.zip" + standalone_path: "naomi2.zip" system: sega-naomi2 required: true category: bios_zip @@ -259,6 +289,7 @@ files: # ------------------------------------------------------- - name: "awbios.zip" path: "dc/awbios.zip" + standalone_path: "awbios.zip" system: sega-atomiswave required: true category: bios_zip @@ -274,6 +305,7 @@ files: # ------------------------------------------------------- - name: "segasp.zip" path: "dc/segasp.zip" + standalone_path: "segasp.zip" system: sega-systemsp required: true category: bios_zip diff --git a/platforms/batocera.yml b/platforms/batocera.yml index ea5f5912..3cab2c10 100644 --- a/platforms/batocera.yml +++ b/platforms/batocera.yml @@ -6111,6 +6111,14 @@ launchers: fails_when_missing: false depends_on_option: gsplus_bios_filename source_ref: "package/batocera/core/batocera-configgen/configgen/configgen/generators/gsplus/gsplusGenerator.py:105" +- emulator: flycast + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/dc + passes_to: environment FLYCAST_BIOS_PATH + fails_when_missing: false + source_ref: "package/batocera/core/batocera-configgen/configgen/configgen/generators/flycast/flycastGenerator.py:199" - emulator: o2em build: libretro kind: core_option diff --git a/platforms/emudeck.yml b/platforms/emudeck.yml index e2db9d80..5ef9d2eb 100644 --- a/platforms/emudeck.yml +++ b/platforms/emudeck.yml @@ -66,6 +66,14 @@ standalone_cores: - xenia - xenianative - yuzu +dual_mode_cores: +- flycast +- mame +- melonds +- mgba +- mupen64plus_next +- ppsspp +- scummvm systems: sony-playstation: files: diff --git a/platforms/rocknix.yml b/platforms/rocknix.yml index 28da7df6..7156ead1 100644 --- a/platforms/rocknix.yml +++ b/platforms/rocknix.yml @@ -757,6 +757,15 @@ launchers: source_ref: "projects/ROCKNIX/packages/emulators/standalone/armsx2-sa/config/S922X/ARMSX2/inis/PCSX2.ini:21,\ \ 475, projects/ROCKNIX/packages/emulators/standalone/armsx2-sa/config/inputplumber/ARMSX2/inis/PCSX2.ini:21,\ \ 473, projects/ROCKNIX/packages/emulators/standalone/armsx2-sa/package.mk:117-118" +- emulator: flycast + build: standalone + kind: passes_path + names: [] + search_dir: /storage/roms/dreamcast/data + passes_to: "find_user_data_dir() in core/linux-dist/main.cpp returns it" + fails_when_missing: false + source_ref: "projects/ROCKNIX/packages/emulators/standalone/flycast-sa/patches/000-conf-dir.patch:25,\ + \ 29" - emulator: drastic build: standalone kind: links diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 2bd48c88..dd54993d 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -886,6 +886,13 @@ "type": "string", "description": "Branch the port was read from, when it is not the default one." }, + "standalone_directories": { + "type": "array", + "items": { + "type": "string" + }, + "description": "What the standalone build resolves standalone_path against, as its code names it: a directory ($XDG_DATA_HOME/flycast), a setting (Dreamcast.BiosPath), an environment variable or the function returning the directory. A platform launcher record that links or passes a folder of its BIOS tree under one of these puts the standalone files in that folder." + }, "prose_written_at": { "type": "string", "pattern": "^[0-9a-f]{40}$", diff --git a/scripts/common.py b/scripts/common.py index daf45018..e67f3fb0 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -6,7 +6,7 @@ and file resolution - eliminates DRY violations across scripts. from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Iterable from collections import Counter import hashlib @@ -2249,6 +2249,9 @@ def runs_standalone( # Marks, in a platform's standalone set, an emulator it also launches as a # libretro core. No core name starts with it. DUAL_MODE_PREFIX = "dual:" +# Marks, in the same set, a whole directory the platform's launcher hands a +# standalone emulator (launchers.handed_directories), as JSON. +HANDED_PREFIX = "handed:" def standalone_set(config: dict) -> set[str]: @@ -2260,9 +2263,60 @@ def standalone_set(config: dict) -> set[str]: """ names = {str(c) for c in config.get("standalone_cores") or []} names |= {DUAL_MODE_PREFIX + str(c) for c in config.get("dual_mode_cores") or []} + if names: + from launchers import handed_directories # noqa: PLC0415 + + names |= { + HANDED_PREFIX + json.dumps( + [h.emulator, h.folder, h.directory, sorted(h.handles), h.source_ref] + ) + for h in handed_directories(config) + } return names +def _handed(standalone_cores: Iterable[str]) -> list: + """The directories a standalone set carries, as launchers built them.""" + from launchers import HandedDirectory # noqa: PLC0415 + + found = [] + for tag in standalone_cores: + if not tag.startswith(HANDED_PREFIX): + continue + emulator, folder, directory, handles, ref = json.loads(tag[len(HANDED_PREFIX):]) + found.append(HandedDirectory(emulator, folder, directory, frozenset(handles), ref)) + return found + + +def standalone_destination(entry: dict, root: str | None) -> str: + """Where a standalone build reads an entry, relative to the BIOS root. + + ``standalone_path`` is relative to the directories the build resolves + it against (``standalone_directories``). A platform that hands one of + them a folder of its BIOS tree puts the file under that folder: + RetroPie and ROCKNIX give Flycast bios/dc, EmuDeck bios/flycast/bios. + An entry with no ``standalone_path`` keeps ``path``, the layout both + builds share. + """ + own = entry.get("standalone_path") + if own and root: + return f"{root}/{own}" + return own or entry.get("path") or entry.get("name", "") + + +def standalone_root_for( + emu_name: str, profile: dict, standalone_cores: Iterable[str] +) -> str | None: + """The folder a platform gives this emulator's standalone layout, or None.""" + declared = profile.get("standalone_directories") or [] + if not declared: + return None + from launchers import standalone_root # noqa: PLC0415 + + names = {emu_name, *(str(c) for c in profile.get("cores") or [])} + return standalone_root(names, declared, _handed(standalone_cores)) + + def platform_layouts( emu_name: str, profile: dict, standalone_cores: set[str] ) -> list[tuple[dict, bool]]: @@ -2275,13 +2329,27 @@ def platform_layouts( """ if not runs_standalone(emu_name, profile, standalone_cores): return [(profile, False)] + standalone = profile + root = standalone_root_for(emu_name, profile, standalone_cores) + if root: + # The standalone view carries the platform's destination, so every + # reader of standalone_path (builder, manifest, verify, slots) places + # the file in the folder the platform hands the emulator. + standalone = { + **profile, + "files": [ + {**f, "standalone_path": standalone_destination(f, root)} + if isinstance(f, dict) and f.get("standalone_path") else f + for f in profile.get("files") or [] + ], + } dual = {c.removeprefix(DUAL_MODE_PREFIX) for c in standalone_cores if c.startswith(DUAL_MODE_PREFIX)} if "libretro" in str(profile.get("type", "")) and dual & { emu_name, *(str(c) for c in profile.get("cores", [])) }: - return [(profile, False), (profile, True)] - return [(profile, True)] + return [(profile, False), (standalone, True)] + return [(standalone, True)] # What identifies a profile entry's content, carried unchanged wherever the diff --git a/scripts/launchers.py b/scripts/launchers.py index ae81146a..d3147b35 100644 --- a/scripts/launchers.py +++ b/scripts/launchers.py @@ -202,6 +202,134 @@ def launcher_places(config: dict) -> list[LauncherPlace]: return places +# The names a platform's launch code gives its BIOS root, besides the +# base_destination _bios_relative reads: RetroPie's scriptmodules set +# biosdir="$home/RetroPie/BIOS", EmuDeck's scripts ${biosPath}, RetroDECK's +# component scripts $bios_path. +BIOS_ROOT_VARIABLES = ("$biosdir", "$home/RetroPie/BIOS", "${biosPath}", "$bios_path") + +# The kinds that hand an emulator a whole directory. +HANDED_KINDS = ("links", "passes_path") + +_HOME = re.compile(r"^(?:\$\{HOME\}|\$HOME|\$home|~)(?=/|$)") +# Flatpak sets XDG_DATA_HOME to ~/.var/app//data inside the sandbox. +_FLATPAK_DATA = re.compile(r"^~/\.var/app/[^/]+/data(?=/|$)") + + +def normalize_handle(token: str) -> str: + """One spelling for a path, setting or function a launcher record names. + + ``$home``, ``$HOME``, ``${HOME}`` and ``~`` are one directory, and a + Flatpak's data directory is its ``$XDG_DATA_HOME``. + """ + token = token.strip().strip("\"'").rstrip(",;").rstrip("/") + token = token.removesuffix("()") + token = _HOME.sub("~", token) + return _FLATPAK_DATA.sub("$XDG_DATA_HOME", token) + + +def _folder_in_bios(directory: str, base: str) -> str | None: + """A directory relative to the BIOS root, None when it lies outside.""" + relative = _bios_relative(directory, base) + if relative is not None: + return relative + directory = directory.strip().rstrip("/") + for variable in BIOS_ROOT_VARIABLES: + if directory == variable: + return "" + if directory.startswith(variable + "/"): + rest = directory[len(variable) + 1:] + return None if re.search(r"[\s<>$:;]", rest) else rest + return None + + +@dataclass(frozen=True) +class HandedDirectory: + """A whole directory a platform hands a standalone emulator. + + ``folder`` is the directory relative to the BIOS root, None when it lies + outside it. ``handles`` are the paths, settings and functions the record + names on the emulator's side: the link's other end, the config key, the + function a patch rewrites. + """ + + emulator: str + folder: str | None + directory: str + handles: frozenset[str] + source_ref: str + + +def handed_directories(config: dict) -> list[HandedDirectory]: + """Every whole directory a platform links or passes to a standalone build. + + A record naming files (gpsp's gba_bios.bin) or one slot of a directory + (RetroDECK's GC/EUR) hands over no whole directory, and a search_dir + holding several directories names no single one. + """ + base = str(config.get("base_destination") or "bios") + found: list[HandedDirectory] = [] + for record in config.get("launchers") or []: + if record.get("build") != "standalone" or record.get("kind") not in HANDED_KINDS: + continue + if record.get("names") or record.get("slot"): + continue + raw = str(record.get("search_dir") or "").strip() + if not raw or re.search(r"[\s,:;]", raw): + continue + directory = normalize_handle(raw) + handles = frozenset( + handle + for token in str(record.get("passes_to") or "").split() + if (handle := normalize_handle(token)) and handle != directory + ) + for emulator in (e.strip() for e in str(record.get("emulator", "")).split(",")): + found.append(HandedDirectory( + emulator=emulator, + folder=_folder_in_bios(raw, base), + directory=directory, + handles=handles, + source_ref=str(record.get("source_ref", "")), + )) + return found + + +def standalone_root( + names: Iterable[str], declared: Iterable[str], handed: Iterable[HandedDirectory] +) -> str | None: + """The folder of the BIOS tree a platform gives a standalone build's layout. + + ``declared`` are the directories the build resolves ``standalone_path`` + against, as its profile names them (``standalone_directories``). A + record handing a folder of the BIOS tree under one of them gives it + directly: RetroPie points Flycast's ~/.local/share/flycast at bios/dc. A + record handing a directory outside the tree under one of them is + followed through a link to the tree: ROCKNIX's patched Flycast uses + /storage/roms/dreamcast/data, which its start script links to bios/dc. + + None when nothing gives a folder, or when two records give two: the + layout is then the profile's, relative to the BIOS root. + """ + names = set(names) + wanted = {normalize_handle(d) for d in declared} + if not wanted: + return None + records = [h for h in handed if h.emulator in names] + folders: set[str] = set() + for record in records: + if not record.handles & wanted: + continue + if record.folder is not None: + folders.add(record.folder) + continue + folders.update( + link.folder + for link in records + if link.folder is not None and record.directory in link.handles + ) + return next(iter(folders)) if len(folders) == 1 else None + + def declared_destinations(config: dict) -> set[str]: """Each destination the platform declares, relative to the core file directory. diff --git a/scripts/scraper/batocera_launchers.py b/scripts/scraper/batocera_launchers.py index c53edc6a..eb11b148 100644 --- a/scripts/scraper/batocera_launchers.py +++ b/scripts/scraper/batocera_launchers.py @@ -647,6 +647,46 @@ def gsplus(cg: Configgen) -> list[Launcher]: )] +# Generators that hand a directory of /userdata/bios to the emulator through +# its environment, with the module holding their path constants. +ENVIRONMENT_GENERATORS = (("flycast.flycastGenerator", "flycast/flycastPaths.py"),) + + +def environment_directories(cg: Configgen) -> list[Launcher]: + """Command(..., env={KEY: }) in the generator.""" + launchers = [] + for module, paths_module in ENVIRONMENT_GENERATORS: + src = cg.generator(module) + constants = dict(cg.paths) + for node in cg.source(f"{GENERATORS}/{paths_module}").tree.body: + name = _assigned_name(node) + value = _path_value(getattr(node, "value", None), constants) + if name is not None and value is not None: + constants[name] = value + prefix = cg.bios.rstrip("/") + found = [ + (key.value, value, key) + for call in _calls(src.tree, attr="Command") + for keyword in call.keywords + if keyword.arg == "env" and isinstance(keyword.value, ast.Dict) + for key, item in zip(keyword.value.keys, keyword.value.values) + if isinstance(key, ast.Constant) + and (value := _path_value(item, constants)) is not None + and (value == prefix or value.startswith(prefix + "/")) + ] + if not found: + src.fail("the command environment names no /userdata/bios directory") + launchers.extend( + Launcher( + emulator=cg.emulator(module), build="standalone", kind="passes_path", + names=[], search_dir=value, passes_to=f"environment {key}", + source_ref=src.ref([(node.lineno, node.end_lineno)]), + ) + for key, value, node in found + ) + return launchers + + # libretro core options whose value names the BIOS file the core opens. FILE_OPTIONS = (("_o2em_options", "o2em_bios"), ("_neocd_options", "neocd_bios")) @@ -701,7 +741,7 @@ def core_file_options(cg: Configgen) -> list[Launcher]: EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = ( duckstation, duckstation_legacy, hatari, xemu, melonds, mupen64plus, - x16emu, gsplus, core_file_options, + x16emu, gsplus, environment_directories, core_file_options, ) diff --git a/scripts/scraper/emudeck_scraper.py b/scripts/scraper/emudeck_scraper.py index 75381a5b..4f72adef 100644 --- a/scripts/scraper/emudeck_scraper.py +++ b/scripts/scraper/emudeck_scraper.py @@ -231,6 +231,25 @@ def launcher_revision(url: str) -> str: return found.group(1) +RETROARCH_SCRIPT = "functions/EmuScripts/emuDeckRetroArch.sh" + + +def installed_libretro_cores(script: str) -> list[str]: + """The cores RetroArch_installCores downloads, without the _libretro suffix.""" + body = re.search(r"RetroArch_installCores\(\)\s*\{(.*?)\n\}", script, re.S) + array = re.search(r"RAcores=\((.*?)\)", body.group(1), re.S) if body else None + if array is None: + raise RuntimeError(f"{RETROARCH_SCRIPT}: no RAcores list in RetroArch_installCores") + cores = [ + word.removesuffix("_libretro") + for word in array.group(1).replace("\\", " ").split() + if word.endswith("_libretro") + ] + if not cores: + raise RuntimeError(f"{RETROARCH_SCRIPT}: RAcores names no core") + return cores + + class Scraper(BaseScraper): """Scraper for EmuDeck checkBIOS.sh and CSV cheat sheets.""" @@ -451,6 +470,9 @@ class Scraper(BaseScraper): version = fetch_github_latest_version("dragoonDorise/EmuDeck") cores = self._fetch_installed_emulators() + standalone = [core for core in cores if core != "retroarch"] + tree = RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url)) + libretro = set(installed_libretro_cores(tree.read(RETROARCH_SCRIPT))) return { "platform": "EmuDeck", @@ -469,12 +491,14 @@ class Scraper(BaseScraper): # Dolphin, Cemu, xemu, ARMSX2 and Supermodel ran with the libretro # layout and lost their standalone-only files. A profile with no # standalone build keeps its layout (runs_standalone). - "standalone_cores": [core for core in cores if core != "retroarch"], + "standalone_cores": standalone, + # RetroArch_installCores downloads a libretro build of some of + # them (flycast_libretro reads system/dc/ beside the standalone + # Flycast's flycast/bios/): both layouts are packed. + "dual_mode_cores": [core for core in standalone if core in libretro], "systems": systems, # Read at the commit checkBIOS.sh comes from. - "launchers": extract_launchers( - RepoTree.github(EMUDECK_REPO, launcher_revision(self.checkbios_url)) - ), + "launchers": extract_launchers(tree), } def _fetch_installed_emulators(self) -> list[str]: diff --git a/scripts/scraper/rocknix_launchers.py b/scripts/scraper/rocknix_launchers.py index 55a8a5de..8aed2554 100644 --- a/scripts/scraper/rocknix_launchers.py +++ b/scripts/scraper/rocknix_launchers.py @@ -612,6 +612,52 @@ def links(tree: RepoTree) -> list[Launcher]: return records +# Packages whose build patch replaces the function an emulator resolves a +# directory with by a fixed path: ROCKNIX's Flycast returns +# /storage/roms/dreamcast/data as its data directory, which start_flycast.sh +# then links to the BIOS tree. +DIRECTORY_PATCHES = ( + ("flycast", f"{STANDALONE}/flycast-sa/patches/000-conf-dir.patch", "find_user_data_dir"), +) + + +def patched_directories(tree: RepoTree) -> list[Launcher]: + records = [] + for emulator, path, function in DIRECTORY_PATCHES: + lines = tree.read(path).splitlines() + target = next( + (line[len("+++ b/"):] for line in lines if line.startswith("+++ b/")), None + ) + start = next( + (n for n, line in enumerate(lines, 1) if re.search(rf"\b{function}\(\)", line)), + None, + ) + returned = None + if target is not None and start is not None: + returned = next( + ( + (n, match.group(1)) + for n, line in enumerate(lines[start:], start + 1) + if (match := re.match(r'^\+\s*return "(/[^"]+)";', line)) + ), + None, + ) + if returned is None: + raise LauncherPatternMissing(path, f"{function}() patched to return a fixed path") + number, directory = returned + records.append(Launcher( + emulator=emulator, + build="standalone", + kind="passes_path", + names=[], + source_ref=source_ref([(path, {start, number})]), + search_dir=directory.rstrip("/"), + passes_to=f"{function}() in {target} returns it", + fails_when_missing=False, + )) + return records + + def _existence_guard(script: Script, link: Link) -> int | None: """The line of a -e test of the BIOS-side path the link waits for, if any.""" last = max(link.lines) @@ -639,6 +685,7 @@ EXTRACTORS: tuple[Callable[[RepoTree], list[Launcher]], ...] = ( duckstation, aethersx2, armsx2, + patched_directories, links, ) diff --git a/tests/test_flycast_layout.py b/tests/test_flycast_layout.py new file mode 100644 index 00000000..81e60b69 --- /dev/null +++ b/tests/test_flycast_layout.py @@ -0,0 +1,31 @@ +"""Flycast's two builds read the same files from two directories. + +The libretro port looks in /dc/ (shell/libretro/libretro.cpp:2249-2250). +The standalone emulator looks in its data directory, whose root holds the +files with no dc/ level (core/oslib/oslib.cpp:118-196, core/stdclass.cpp:95-125). +Without standalone_path the standalone layout of a pack fell back on path: +and put every file one directory too deep. +""" + +from __future__ import annotations + +import unittest +from pathlib import Path + +import yaml + +PROFILE = Path(__file__).resolve().parent.parent / "emulators" / "flycast.yml" + + +class FlycastStandaloneLayout(unittest.TestCase): + def test_every_entry_has_the_data_directory_layout(self) -> None: + profile = yaml.safe_load(PROFILE.read_text(encoding="utf-8")) + self.assertEqual(profile["type"], "standalone + libretro") + for entry in profile["files"]: + with self.subTest(entry=entry["name"]): + self.assertTrue(entry["path"].startswith("dc/")) + self.assertEqual(entry.get("standalone_path"), entry["path"][len("dc/"):]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_standalone_links.py b/tests/test_standalone_links.py new file mode 100644 index 00000000..9d8044fc --- /dev/null +++ b/tests/test_standalone_links.py @@ -0,0 +1,353 @@ +"""A standalone build's files land in the folder its platform hands it. + +``standalone_path`` is relative to the directories the standalone build +resolves it against (``standalone_directories``). RetroPie links Flycast's +~/.local/share/flycast to bios/dc and writes Dreamcast.BiosPath, ROCKNIX +patches Flycast's data directory to /storage/roms/dreamcast/data and links +that to bios/dc, EmuDeck links the Flatpak's data directory to +bios/flycast/bios, Batocera sets FLYCAST_BIOS_PATH and RetroBat +Dreamcast.BiosPath to bios/dc. Laid out at the BIOS root instead, Flycast's +naomi.zip took the place of MAME's. +""" + +from __future__ import annotations + +import sys +import textwrap +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from common import ( # noqa: E402 + load_emulator_profiles, + load_platform_config, + platform_layouts, + standalone_destination, + standalone_root_for, + standalone_set, +) +from launchers import handed_directories, normalize_handle, standalone_root # noqa: E402 + + +def link(emulator: str, search_dir: str, passes_to: str, **fields: object) -> dict: + return { + "emulator": emulator, "build": "standalone", "kind": "links", "names": [], + "search_dir": search_dir, "passes_to": passes_to, "source_ref": "x.sh:1", + **fields, + } + + +def passes(emulator: str, search_dir: str, passes_to: str) -> dict: + return {**link(emulator, search_dir, passes_to), "kind": "passes_path"} + + +PROFILE = { + "emulator": "Emu", + "type": "standalone + libretro", + "cores": ["emu_core"], + "standalone_directories": ["$XDG_DATA_HOME/emu", "$HOME/.local/share/emu", "find_user_data_dir"], + "files": [ + {"name": "boot.bin", "path": "dc/boot.bin", "standalone_path": "boot.bin"}, + {"name": "set.zip", "path": "dc/set.zip", "standalone_path": "set.zip"}, + {"name": "shared.bin", "path": "dc/shared.bin"}, + ], +} + + +def config(*launchers: dict, base: str = "bios") -> dict: + return {"base_destination": base, "standalone_cores": ["emu"], "launchers": list(launchers)} + + +def standalone_view(cfg: dict) -> dict: + views = platform_layouts("emu", PROFILE, standalone_set(cfg)) + return next(view for view, standalone in views if standalone) + + +class HandleSpelling(unittest.TestCase): + def test_home_and_flatpak_spellings_are_one_directory(self) -> None: + self.assertEqual(normalize_handle("$home/.local/share/emu"), "~/.local/share/emu") + self.assertEqual(normalize_handle("${HOME}/.local/share/emu/"), "~/.local/share/emu") + self.assertEqual( + normalize_handle("${HOME}/.var/app/org.emu.Emu/data/emu/"), "$XDG_DATA_HOME/emu" + ) + self.assertEqual(normalize_handle("find_user_data_dir()"), "find_user_data_dir") + + +class HandedDirectories(unittest.TestCase): + def test_retropie_variable_resolves_to_a_bios_folder(self) -> None: + found = handed_directories(config( + link("emu", "$biosdir/dc", "$home/.local/share/emu is a symlink to it"), base="BIOS", + )) + self.assertEqual([h.folder for h in found], ["dc"]) + self.assertIn("~/.local/share/emu", found[0].handles) + + def test_named_and_slotted_links_hand_no_directory(self) -> None: + found = handed_directories(config( + {**link("emu", "$biosdir", "$md_inst/"), "names": ["boot.bin"]}, + {**link("emu", "$bios_path/gc/EU", "$XDG_DATA_HOME/emu/GC/EUR"), "slot": "EUR"}, + )) + self.assertEqual(found, []) + + def test_directory_outside_the_tree_has_no_folder(self) -> None: + found = handed_directories(config( + passes("emu", "/storage/roms/dreamcast/data", "find_user_data_dir() returns it"), + )) + self.assertIsNone(found[0].folder) + + +class Root(unittest.TestCase): + def root(self, *launchers: dict, base: str = "bios") -> str | None: + return standalone_root( + {"emu"}, PROFILE["standalone_directories"], handed_directories(config(*launchers, base=base)) + ) + + def test_link_of_the_data_directory_gives_its_folder(self) -> None: + self.assertEqual( + self.root(link("emu", "$biosdir/dc", "$home/.local/share/emu is a symlink to it")), + "dc", + ) + + def test_flatpak_data_directory(self) -> None: + self.assertEqual( + self.root(link( + "emu", "${biosPath}/emu/bios", + "symlink ${biosPath}/emu/bios -> ${HOME}/.var/app/org.emu.Emu/data/emu/", + )), + "emu/bios", + ) + + def test_patched_data_directory_followed_through_its_link(self) -> None: + self.assertEqual( + self.root( + passes("emu", "/storage/roms/dreamcast/data", "find_user_data_dir() returns it"), + link("emu", "/storage/roms/bios/dc", + "symlink /storage/roms/dreamcast/data -> /storage/roms/bios/dc"), + ), + "dc", + ) + + def test_link_of_another_directory_gives_nothing(self) -> None: + # A subfolder of the emulator's tree (Dolphin's Triforce) is not the + # directory standalone_path is relative to. + self.assertIsNone(self.root( + link("emu", "$bios_path/Triforce", "$XDG_DATA_HOME/emu/Triforce is a symlink to it"), + )) + + def test_two_folders_give_nothing(self) -> None: + self.assertIsNone(self.root( + link("emu", "$biosdir/dc", "$home/.local/share/emu is a symlink to it"), + link("emu", "$biosdir/other", "$XDG_DATA_HOME/emu is a symlink to it"), + )) + + +class Layout(unittest.TestCase): + LINKED = link("emu", "$biosdir/dc", "$home/.local/share/emu is a symlink to it") + + def test_standalone_view_takes_the_linked_folder(self) -> None: + view = standalone_view(config(self.LINKED, base="BIOS")) + self.assertEqual( + [f.get("standalone_path") for f in view["files"]], ["dc/boot.bin", "dc/set.zip", None] + ) + self.assertEqual(view["files"][2]["path"], "dc/shared.bin") + + def test_without_a_link_the_profile_layout_stands(self) -> None: + view = standalone_view(config()) + self.assertEqual([f.get("standalone_path") for f in view["files"]][:2], ["boot.bin", "set.zip"]) + + def test_profile_is_not_changed(self) -> None: + standalone_view(config(self.LINKED, base="BIOS")) + self.assertEqual(PROFILE["files"][0]["standalone_path"], "boot.bin") + + def test_libretro_view_keeps_path(self) -> None: + cfg = {**config(self.LINKED, base="BIOS"), "dual_mode_cores": ["emu"]} + views = platform_layouts("emu", PROFILE, standalone_set(cfg)) + libretro = next(view for view, standalone in views if not standalone) + self.assertIs(libretro, PROFILE) + + def test_dual_mode_gets_both_layouts(self) -> None: + # The libretro build still reads dc/, the standalone build the + # linked folder: neither loses a file to the other. + cfg = {**config(link("emu", "$biosdir/emu", "$home/.local/share/emu is a symlink to it"), + base="BIOS"), "dual_mode_cores": ["emu"]} + views = dict((standalone, view) for view, standalone in + platform_layouts("emu", PROFILE, standalone_set(cfg))) + self.assertEqual( + [f["path"] for f in views[False]["files"]], ["dc/boot.bin", "dc/set.zip", "dc/shared.bin"] + ) + self.assertEqual( + [f.get("standalone_path") for f in views[True]["files"]], ["emu/boot.bin", "emu/set.zip", None] + ) + + def test_union_of_platforms_with_two_folders_keeps_the_profile_layout(self) -> None: + union = standalone_set(config(self.LINKED, base="BIOS")) | standalone_set(config( + link("emu", "${biosPath}/emu/bios", "symlink ${biosPath}/emu/bios -> ${HOME}/.local/share/emu"), + )) + self.assertIsNone(standalone_root_for("emu", PROFILE, union)) + + def test_destination_function(self) -> None: + entry = PROFILE["files"][0] + self.assertEqual(standalone_destination(entry, "dc"), "dc/boot.bin") + self.assertEqual(standalone_destination(entry, None), "boot.bin") + self.assertEqual(standalone_destination(PROFILE["files"][2], "dc"), "dc/shared.bin") + + +class FlycastOnEachPlatform(unittest.TestCase): + """The real profile against the real launcher records.""" + + EXPECTED = { + "batocera": "dc", + "emudeck": "flycast/bios", + "retrobat": "dc", + "retropie": "dc", + "rocknix": "dc", + } + + @classmethod + def setUpClass(cls) -> None: + cls.profile = load_emulator_profiles(str(REPO_ROOT / "emulators"))["flycast"] + + def test_each_platform_hands_flycast_its_folder(self) -> None: + for platform, folder in self.EXPECTED.items(): + with self.subTest(platform=platform): + cfg = load_platform_config(platform, str(REPO_ROOT / "platforms")) + self.assertEqual( + standalone_root_for("flycast", self.profile, standalone_set(cfg)), folder + ) + + def test_no_standalone_file_lands_at_the_bios_root(self) -> None: + # Flycast's naomi.zip and awbios.zip share their names with MAME's + # root archives: at the root they replace them. + for platform, folder in self.EXPECTED.items(): + with self.subTest(platform=platform): + cfg = load_platform_config(platform, str(REPO_ROOT / "platforms")) + for view, standalone in platform_layouts( + "flycast", self.profile, standalone_set(cfg) + ): + if not standalone: + continue + for entry in view["files"]: + self.assertTrue( + entry["standalone_path"].startswith(folder + "/"), + entry["standalone_path"], + ) + + + def test_emudeck_keeps_the_libretro_layout_beside_the_linked_one(self) -> None: + # EmuDeck's RetroArch_installCores downloads flycast_libretro, which + # reads system/dc/, next to the standalone Flycast in flycast/bios/. + cfg = load_platform_config("emudeck", str(REPO_ROOT / "platforms")) + self.assertIn("flycast", cfg.get("dual_mode_cores") or []) + views = { + standalone: view + for view, standalone in platform_layouts("flycast", self.profile, standalone_set(cfg)) + } + self.assertEqual(set(views), {False, True}) + for entry in views[False]["files"]: + self.assertTrue(entry["path"].startswith("dc/"), entry["path"]) + for entry in views[True]["files"]: + self.assertTrue(entry["standalone_path"].startswith("flycast/bios/")) + + +class Scrapers(unittest.TestCase): + def test_emudeck_installed_libretro_cores(self) -> None: + from scraper.emudeck_scraper import installed_libretro_cores + + script = textwrap.dedent("""\ + RetroArch_installCores(){ + \tRAcores=( + \t\t\t81_libretro \\ + \t\t\tflycast_libretro \\ + \t\t\tmesen-s_libretro \\ + \t\t\t) + } + """) + self.assertEqual(installed_libretro_cores(script), ["81", "flycast", "mesen-s"]) + with self.assertRaises(RuntimeError): + installed_libretro_cores("RetroArch_installCores(){\n}\n") + + """The records each platform's scraper writes for these directories.""" + + def test_rocknix_patched_data_directory(self) -> None: + from scraper import rocknix_launchers as rl + from scraper.shell_launchers import LauncherPatternMissing, RepoTree + + path = f"{rl.STANDALONE}/flycast-sa/patches/000-conf-dir.patch" + patch = textwrap.dedent("""\ + --- a/core/linux-dist/main.cpp + +++ b/core/linux-dist/main.cpp + @@ -69,12 +69,17 @@ + static std::string find_user_data_dir() + { + +#ifdef __unix__ + +\tflycast::mkdir("/storage/roms/dreamcast/data", 0755); + +\treturn "/storage/roms/dreamcast/data/"; + +#else + """) + records = rl.patched_directories(RepoTree.from_files({path: patch})) + self.assertEqual(len(records), 1) + entry = records[0].entry() + self.assertEqual(entry["search_dir"], "/storage/roms/dreamcast/data") + self.assertEqual(entry["passes_to"], "find_user_data_dir() in core/linux-dist/main.cpp returns it") + self.assertEqual(entry["source_ref"], f"{path}:4, 8") + with self.assertRaises(LauncherPatternMissing): + rl.patched_directories(RepoTree.from_files({path: patch.replace("return", "x =")})) + + def test_batocera_environment_directory(self) -> None: + from scraper.batocera_launchers import ( + GENERATORS, IMPORTER, PATHS_MODULE, Configgen, environment_directories, + ) + from scraper.configgen_launchers import LauncherShapeError + + paths = "from pathlib import Path\nBIOS = Path('/userdata/bios')\nCONFIGS = Path('/userdata/system/configs')\n" + importer = "_GENERATOR_MAP = {}\n_LEGACY_GENERATOR_MAP = {}\n" + flycast_paths = "from ...batoceraPaths import BIOS\nFLYCAST_BIOS = BIOS / 'dc'\n" + generator = textwrap.dedent("""\ + def generate(self): + return Command.Command( + array=[], + env={ + "XDG_CONFIG_HOME": CONFIGS, + "FLYCAST_BIOS_PATH": FLYCAST_BIOS, + } + ) + """) + tree = { + PATHS_MODULE: paths, + IMPORTER: importer, + f"{GENERATORS}/flycast/flycastPaths.py": flycast_paths, + f"{GENERATORS}/flycast/flycastGenerator.py": generator, + } + records = [r.entry() for r in environment_directories(Configgen(tree.__getitem__))] + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["search_dir"], "/userdata/bios/dc") + self.assertEqual(records[0]["passes_to"], "environment FLYCAST_BIOS_PATH") + self.assertTrue(records[0]["source_ref"].endswith("flycastGenerator.py:6")) + tree[f"{GENERATORS}/flycast/flycastGenerator.py"] = generator.replace("FLYCAST_BIOS,", "CONFIGS,") + with self.assertRaises(LauncherShapeError): + environment_directories(Configgen(tree.__getitem__)) + + def test_retrobat_bios_path_setting(self) -> None: + from scraper import retrobat_launchers as rl + + path = f"{rl.GENERATORS}/Flycast.Generator.cs" + text = textwrap.dedent("""\ + private void SetupConfiguration(string path) + { + string configfile = Path.Combine(path, "emu.cfg"); + using (var ini = IniFile.FromFile(configfile, IniOptions.UseSpaces)) + { + string biosPath = Path.Combine(AppConfig.GetFullPath("bios"), "dc"); + ini.Remove("config", "Dreamcast.BiosPath"); + ini.WriteValue("config", "Dreamcast.BiosPath", biosPath); + } + } + """) + records = rl._value_writes(rl.Source(path, text), "flycast", "standalone") + self.assertEqual([r["search_dir"] for r in records], ["bios/dc"]) + self.assertIn("Dreamcast.BiosPath", records[0]["passes_to"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index d4d1349a..ee7ed3c8 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -182,7 +182,14 @@ resolved. Three strategies exist: - **Omitted**: fallback to system ID intersection, `built_by` profiles excluded. Two lists say how each resolved emulator is laid out. `standalone_cores` names the -ones the platform launches as standalone apps: their files go to `standalone_path`. +ones the platform launches as standalone apps: their files go to `standalone_path`, +under the folder the platform hands the emulator when its `launchers:` say so. A +`links` or `passes_path` record of a whole directory that names one of the profile's +`standalone_directories` gives that folder (`launchers.standalone_root`): RetroPie +links Flycast's `~/.local/share/flycast` to `BIOS/dc`, EmuDeck the Flatpak's data +directory to `bios/flycast/bios`, and ROCKNIX's patched data directory is followed +through the link its start script makes. Two records giving two folders leave the +profile's layout. `dual_mode_cores` names those it launches both ways (RetroPie packages `lr-hatari` and `hatari`): both layouts are packed. A profile without a standalone build keeps its libretro layout whatever the platform calls it. diff --git a/wiki/profiling.md b/wiki/profiling.md index ebf96a27..726e8620 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -493,6 +493,7 @@ reports the profile as uncheckable, and the CI job that runs | `built_by` | no | platforms that ship this build under its core name, compiled by them or installed from the project's own release (Recalbox's `mame0278`, the GSplus legacy build). On those platforms the name resolves to this profile, never through `all_libretro`: the buildbot does not publish it | | `bios_mode` | no | `agnostic` when the core takes any file of the right shape in its BIOS folder instead of a fixed name (PCSX2). Packs then carry every matching dump, and a `.bin` placeholder stands for them | | `bios_directory` | no | the folder an agnostic core scans, relative to the system directory (`pcsx2/bios/`). A scan never seeds from files outside it | +| `standalone_directories` | no | what the standalone build resolves `standalone_path` against, as its code names it: a directory (`$XDG_DATA_HOME/flycast`), a setting (`Dreamcast.BiosPath`), an environment variable (`FLYCAST_BIOS_PATH`) or the function returning the directory (`find_user_data_dir`). A platform whose launcher links or passes a folder of its BIOS tree under one of them gets the standalone files in that folder | | `display_name` | no | full display name (e.g. "Sega - Mega Drive (BlastEm)") | | `logo` | no | image URL used on the emulator page | | `systems` | no | list of system IDs this core handles | diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 52c0d41e..b3a50f9c 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -39,7 +39,7 @@ Every module in `tests/`, by the area it guards. |------|----------|---------| | Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` | | File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` | -| Verification, cross-reference and profile checks | synthetic + profiles | `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_file_consumers`, `test_fileless_audit`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | +| Verification, cross-reference and profile checks | synthetic + profiles | `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` | | Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |