From 6a9700489d81f720403360b5cf1c9b51bbaa9001 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:28:00 +0100 Subject: [PATCH] feat: place standalone files through links standalone_bindings maps each subfolder of a standalone layout to the folder of the BIOS tree a platform's links and passed directories give it, followed through a working directory. platform_layouts applies it, so the pack, the manifest, verify and slots read one destination. uae4all paths are now what the emulator reads, kickstarts/ below its working directory, and RetroPie still receives them in amiga/. --- emulators/uae4all.yml | 24 +++-- scripts/common.py | 66 +++++++------ scripts/launchers.py | 105 +++++++++++++++------ tests/test_launcher_placement.py | 154 +++++++++++++++++++++++++++++++ tests/test_standalone_links.py | 37 +++++--- 5 files changed, 308 insertions(+), 78 deletions(-) create mode 100644 tests/test_launcher_placement.py diff --git a/emulators/uae4all.yml b/emulators/uae4all.yml index 739c6bd9f..4aacff996 100644 --- a/emulators/uae4all.yml +++ b/emulators/uae4all.yml @@ -16,6 +16,8 @@ display_name: "Commodore - Amiga (UAE4All2)" cores: [uae4all] systems: [commodore-amiga] verification: existence +standalone_directories: + - "getcwd" notes: | UAE4All2 is Lubomyr's Android and Pandora Amiga emulator. RetroPie builds @@ -26,9 +28,11 @@ notes: | com.cloanto.amigaforever.essentials is therefore compiled out. AndroidAppSettings.cfg:178 names the version 2.3.6.2. - The emulator runs from its install directory and takes it as launchDir - (src/main.cpp:219). Kickstarts are read from launchDir/kickstarts/, which - RetroPie links to the BIOS directory's amiga/ folder + launchDir is the working directory (getcwd, src/main.cpp:219), and the + Kickstarts and rom.key are read from launchDir/kickstarts/ + (src/main.cpp:95-111). Paths below are relative to the working directory. + RetroPie starts the emulator in its install directory and links + kickstarts/ there to the BIOS directory's amiga/ folder (scriptmodules/emulators/uae4all.sh, configure_uae4all). The kickstart choice indexes kickstarts_rom_names and extended_rom_names @@ -54,35 +58,35 @@ notes: | files: - name: "kick13.rom" - path: "amiga/kick13.rom" + path: "kickstarts/kick13.rom" system: commodore-amiga description: "Kickstart 1.3, the default selection" required: false source_ref: "upstream src/memory.cpp:34, source src/main.cpp:1, source src/main.cpp:95-103" - name: "kick12.rom" - path: "amiga/kick12.rom" + path: "kickstarts/kick12.rom" system: commodore-amiga description: "Kickstart 1.2" required: false source_ref: "upstream src/memory.cpp:34" - name: "kick20.rom" - path: "amiga/kick20.rom" + path: "kickstarts/kick20.rom" system: commodore-amiga description: "Kickstart 2.0" required: false source_ref: "upstream src/memory.cpp:34" - name: "kick31.rom" - path: "amiga/kick31.rom" + path: "kickstarts/kick31.rom" system: commodore-amiga description: "Kickstart 3.1" required: false source_ref: "upstream src/memory.cpp:34" - name: "aros-amiga-m68k-rom.bin" - path: "amiga/aros-amiga-m68k-rom.bin" + path: "kickstarts/aros-amiga-m68k-rom.bin" bundled: true not_bundled_on: [recalbox] system: commodore-amiga @@ -96,7 +100,7 @@ files: note: "Hashes are of the file in AndroidData/aros20140110.zip in the source tree. RetroPie unpacks that zip into the BIOS folder on install." - name: "aros-amiga-m68k-ext.bin" - path: "amiga/aros-amiga-m68k-ext.bin" + path: "kickstarts/aros-amiga-m68k-ext.bin" bundled: true not_bundled_on: [recalbox] system: commodore-amiga @@ -110,7 +114,7 @@ files: note: "Loaded with the AROS main ROM only. Hashes are of the file in AndroidData/aros20140110.zip in the source tree." - name: "rom.key" - path: "amiga/rom.key" + path: "kickstarts/rom.key" system: commodore-amiga description: "Cloanto key for AMIROMTYPE1 encrypted Kickstarts" required: false diff --git a/scripts/common.py b/scripts/common.py index 968320769..2a9088c34 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -2729,33 +2729,48 @@ def _handed(standalone_cores: Iterable[str]) -> list: return found -def standalone_destination(entry: dict, root: str | None) -> str: - """Where a standalone build reads an entry, relative to the BIOS root. +def standalone_layout_path(entry: dict, profile: dict) -> str | None: + """The path of an entry relative to the directories its standalone build declares. - ``standalone_path`` is relative to the directories the build resolves - it against (``standalone_directories``). A platform that hands one of - them a folder of its BIOS tree puts the file under that folder: - RetroPie and ROCKNIX give Flycast bios/dc, EmuDeck bios/flycast/bios. - An entry with no ``standalone_path`` keeps ``path``, the layout both - builds share. + ``standalone_path`` when the entry has one. A profile with no libretro + build has one layout, so its ``path`` is that layout: UAE4All2 reads + kickstarts/kick13.rom below its working directory. """ own = entry.get("standalone_path") - if own and root: - return f"{root}/{own}" - return own or entry.get("path") or entry.get("name", "") + if own: + return own + if "libretro" in str(profile.get("type", "")): + return None + return entry.get("path") or None -def standalone_root_for( +def standalone_destination(entry: dict, profile: dict, bindings: dict[str, str]) -> str | None: + """Where a platform puts a standalone entry, relative to the BIOS root. + + ``bindings`` are the folders the platform's launchers give the layout + (launchers.standalone_bindings): RetroPie and ROCKNIX give Flycast bios/dc, + EmuDeck bios/flycast/bios, RetroPie gives UAE4All2's kickstarts/ bios/amiga. + None when no binding covers the entry: it keeps the profile's layout. + """ + layout = standalone_layout_path(entry, profile) + if layout is None or not bindings: + return None + from launchers import bound_path # noqa: PLC0415 + + return bound_path(layout, bindings) + + +def standalone_bindings_for( emu_name: str, profile: dict, standalone_cores: Iterable[str] -) -> str | None: - """The folder a platform gives this emulator's standalone layout, or None.""" +) -> dict[str, str]: + """The folders a platform gives this emulator's standalone layout, by subfolder.""" declared = profile.get("standalone_directories") or [] if not declared: - return None - from launchers import standalone_root # noqa: PLC0415 + return {} + from launchers import standalone_bindings # noqa: PLC0415 names = {emu_name, *(str(c) for c in profile.get("cores") or [])} - return standalone_root(names, declared, _handed(standalone_cores)) + return standalone_bindings(names, declared, _handed(standalone_cores)) def platform_layouts( @@ -2774,19 +2789,16 @@ def platform_layouts( return [(profile, False)] return [(system_dir_view(profile, d), False) for d in directories] standalone = profile - root = standalone_root_for(emu_name, profile, standalone_cores) - if root: + bindings = standalone_bindings_for(emu_name, profile, standalone_cores) + if bindings: # The standalone view carries the platform's destination, so every # reader of standalone_path (builder, manifest, verify, slots) places # the file in the folder the platform hands the emulator. - standalone = { - **profile, - "files": [ - {**f, "standalone_path": standalone_destination(f, root)} - if isinstance(f, dict) and f.get("standalone_path") else f - for f in profile.get("files") or [] - ], - } + files = [] + for f in profile.get("files") or []: + placed = standalone_destination(f, profile, bindings) if isinstance(f, dict) else None + files.append({**f, "standalone_path": placed} if placed else f) + standalone = {**profile, "files": files} dual = {c.removeprefix(DUAL_MODE_PREFIX) for c in standalone_cores if c.startswith(DUAL_MODE_PREFIX)} if "libretro" in str(profile.get("type", "")) and dual & { diff --git a/scripts/launchers.py b/scripts/launchers.py index 1f9ce1d56..9dd0be254 100644 --- a/scripts/launchers.py +++ b/scripts/launchers.py @@ -211,6 +211,12 @@ BIOS_ROOT_VARIABLES = ("$biosdir", "$home/RetroPie/BIOS", "${biosPath}", "$bios_ # The kinds that hand an emulator a whole directory. HANDED_KINDS = ("links", "passes_path") +# How a launcher record says the platform starts the emulator in a directory +# (retropie_launchers._working_directory), and the handle a profile gives +# that directory in standalone_directories: the call its code reads it with. +WORKING_DIRECTORY_PASSED = "working directory of " +WORKING_DIRECTORY = "getcwd" + _HOME = re.compile(r"^(?:\$\{HOME\}|\$HOME|\$home|~)(?=/|$)") # Flatpak sets XDG_DATA_HOME to ~/.var/app//data inside the sandbox. _FLATPAK_DATA = re.compile(r"^~/\.var/app/[^/]+/data(?=/|$)") @@ -278,11 +284,17 @@ def handed_directories(config: dict) -> list[HandedDirectory]: if not raw or re.search(r"[\s,:;]", raw): continue directory = normalize_handle(raw) - handles = frozenset( - handle - for token in str(record.get("passes_to") or "").split() - if (handle := normalize_handle(token)) and handle != directory - ) + passes_to = str(record.get("passes_to") or "") + if passes_to.startswith(WORKING_DIRECTORY_PASSED): + # The program runs from the directory: what its code reads + # through getcwd(), or relative to it, is below it. + handles = frozenset({WORKING_DIRECTORY}) + else: + handles = frozenset( + handle + for token in passes_to.split() + if (handle := normalize_handle(token)) and handle != directory + ) for emulator in (e.strip() for e in str(record.get("emulator", "")).split(",")): found.append(HandedDirectory( emulator=emulator, @@ -294,40 +306,75 @@ def handed_directories(config: dict) -> list[HandedDirectory]: return found -def standalone_root( +def standalone_bindings( names: Iterable[str], declared: Iterable[str], handed: Iterable[HandedDirectory] -) -> str | None: - """The folder of the BIOS tree a platform gives a standalone build's layout. +) -> dict[str, str]: + """The folders of the BIOS tree a platform gives a standalone build's layout. - ``declared`` are the directories the build resolves ``standalone_path`` - against, as its profile names them (``standalone_directories``). A - record handing a folder of the BIOS tree under one of them gives it - directly: RetroPie points Flycast's ~/.local/share/flycast at bios/dc. A - record handing a directory outside the tree under one of them is - followed through a link to the tree: ROCKNIX's patched Flycast uses - /storage/roms/dreamcast/data, which its start script links to bios/dc. + ``declared`` are the directories the build resolves its paths against, + as its profile names them (``standalone_directories``). The result maps + a subfolder of that layout ("" for the whole of it) to the folder of the + BIOS tree the emulator reads it from. - None when nothing gives a folder, or when two records give two: the - layout is then the profile's, relative to the BIOS root. + A record handing a folder of the tree under a declared directory gives + it directly: RetroPie points Flycast's ~/.local/share/flycast at bios/dc. + A record handing a directory outside the tree is followed through the + links that put a folder of the tree at it or below it: ROCKNIX's patched + Flycast uses /storage/roms/dreamcast/data, which its start script links + to bios/dc, and RetroPie starts UAE4All2 in $md_inst, whose kickstarts + folder it links to bios/amiga. + + A subfolder two records give two folders is left out: its files keep + the profile's layout, relative to the BIOS root. """ names = set(names) wanted = {normalize_handle(d) for d in declared} if not wanted: - return None + return {} records = [h for h in handed if h.emulator in names] - folders: set[str] = set() + + def below(handle: str, directory: str) -> str | None: + if handle == directory: + return "" + if handle.startswith(directory + "/"): + return handle[len(directory) + 1:] + return None + + found: dict[str, set[str]] = {} for record in records: - if not record.handles & wanted: + subs = { + sub for handle in record.handles for directory in wanted + if (sub := below(handle, directory)) is not None + } + for sub in subs: + if record.folder is not None: + found.setdefault(sub, set()).add(record.folder) + continue + for link in records: + if link.folder is None: + continue + for handle in link.handles: + rest = below(handle, record.directory) + if rest is not None: + key = "/".join(p for p in (sub, rest) if p) + found.setdefault(key, set()).add(link.folder) + return {sub: next(iter(folders)) for sub, folders in found.items() if len(folders) == 1} + + +def bound_path(path: str, bindings: dict[str, str]) -> str | None: + """Where the BIOS tree holds a layout path, None when no binding covers it. + + The longest subfolder that contains the path wins: a link of + kickstarts/ places kickstarts/kick13.rom even when the whole layout is + bound elsewhere. + """ + for sub in sorted(bindings, key=len, reverse=True): + if sub and not path.startswith(sub + "/"): continue - if record.folder is not None: - folders.add(record.folder) - continue - folders.update( - link.folder - for link in records - if link.folder is not None and record.directory in link.handles - ) - return next(iter(folders)) if len(folders) == 1 else None + rest = path[len(sub) + 1:] if sub else path + folder = bindings[sub] + return f"{folder}/{rest}" if folder else rest + return None def held_record(report: dict, db: dict) -> dict | None: diff --git a/tests/test_launcher_placement.py b/tests/test_launcher_placement.py new file mode 100644 index 000000000..b3a61041c --- /dev/null +++ b/tests/test_launcher_placement.py @@ -0,0 +1,154 @@ +"""A platform's launcher moves where an emulator's file lands. + +A profile's ``path`` is where the emulator's code reads a file. UAE4All2 +reads kickstarts/ below its working directory, and RetroPie starts it +in its install directory, whose kickstarts/ it links to BIOS/amiga. The +pack, the manifest and verify all put the file in amiga/ on that platform, +and a platform with no such record keeps the profile's layout. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import hashlib +import io +import shutil +import sys +import tempfile +import unittest +import zipfile +from contextlib import redirect_stdout +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from common import ( # noqa: E402 + _emulator_profiles_cache, + load_emulator_profiles, + load_platform_config, + platform_layouts, + standalone_set, +) +from generate_pack import generate_manifest, generate_pack # noqa: E402 +from verify import verify_platform # noqa: E402 + +WORKING_DIRECTORY = { + "emulator": "kick", "build": "standalone", "kind": "passes_path", "names": [], + "search_dir": "$md_inst", "passes_to": "working directory of $romdir/amiga/+Start.sh", + "source_ref": "kick.sh:90", +} +LINK = { + "emulator": "kick", "build": "standalone", "kind": "links", "names": [], + "search_dir": "$biosdir/amiga", "passes_to": "$md_inst/kickstarts is a symlink to it", + "source_ref": "kick.sh:79", +} + + +class ALinkMovesTheDestination(unittest.TestCase): + def setUp(self) -> None: + _emulator_profiles_cache.clear() + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + self.root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, self.root, True) + bios = self.root / "bios" / "Commodore" / "Amiga" + bios.mkdir(parents=True) + data = b"KICKSTART 1.3" + (bios / "kick13.rom").write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + md5 = hashlib.md5(data).hexdigest() + self.db = { + "files": {sha1: { + "path": str(bios / "kick13.rom"), "name": "kick13.rom", "sha1": sha1, + "md5": md5, "crc32": "", "size": len(data), + }}, + "indexes": { + "by_md5": {md5: sha1}, "by_name": {"kick13.rom": [sha1]}, + "by_crc32": {}, "by_path_suffix": {}, + }, + } + self.platforms = self.root / "platforms" + self.emulators = self.root / "emulators" + self.platforms.mkdir() + self.emulators.mkdir() + for name, launchers in (("linkplat", [WORKING_DIRECTORY, LINK]), ("plainplat", [])): + (self.platforms / f"{name}.yml").write_text(yaml.safe_dump({ + "platform": name, "version": "1", "verification_mode": "existence", + "base_destination": "BIOS", "cores": ["kick"], "standalone_cores": ["kick"], + "launchers": launchers, + "systems": {"commodore-amiga": {"files": []}}, + })) + (self.platforms / "_registry.yml").write_text(yaml.safe_dump({"platforms": { + "linkplat": {"config": "linkplat.yml"}, "plainplat": {"config": "plainplat.yml"}, + }})) + (self.emulators / "kick.yml").write_text(yaml.safe_dump({ + "emulator": "Kick", "type": "standalone", "cores": ["kick"], + "systems": ["commodore-amiga"], + "standalone_directories": ["getcwd"], + "files": [{"name": "kick13.rom", "path": "kickstarts/kick13.rom", + "sha1": sha1, "required": False}], + })) + + def pack_names(self, platform: str) -> list[str]: + with redirect_stdout(io.StringIO()): + zip_path = generate_pack( + platform, str(self.platforms), self.db, str(self.root / "bios"), + str(self.root / "out" / platform), emulators_dir=str(self.emulators), + offline=True, + ) + with zipfile.ZipFile(zip_path) as zf: + return [n for n in zf.namelist() if n.endswith("kick13.rom")] + + def manifest_dests(self, platform: str) -> list[str]: + with redirect_stdout(io.StringIO()): + manifest = generate_manifest( + platform, str(self.platforms), self.db, str(self.root / "bios"), + str(self.platforms / "_registry.yml"), + emulators_dir=str(self.emulators), offline=True, + ) + return [f["dest"] for f in manifest["files"] if f["dest"].endswith("kick13.rom")] + + def verify_paths(self, platform: str) -> list[str]: + config = load_platform_config(platform, str(self.platforms)) + result = verify_platform(config, self.db, str(self.emulators)) + return [u["path"] for u in result["undeclared_files"] if u["name"] == "kick13.rom"] + + def test_the_link_moves_it_in_pack_manifest_and_verify(self) -> None: + self.assertEqual(self.pack_names("linkplat"), ["amiga/kick13.rom"]) + self.assertEqual(self.manifest_dests("linkplat"), ["amiga/kick13.rom"]) + self.assertEqual(self.verify_paths("linkplat"), ["amiga/kick13.rom"]) + + def test_a_platform_without_the_record_is_unchanged(self) -> None: + self.assertEqual(self.pack_names("plainplat"), ["kickstarts/kick13.rom"]) + self.assertEqual(self.manifest_dests("plainplat"), ["kickstarts/kick13.rom"]) + self.assertEqual(self.verify_paths("plainplat"), ["kickstarts/kick13.rom"]) + + def test_the_link_alone_binds_nothing(self) -> None: + # Without the working directory the link's $md_inst is not the + # directory the profile's paths are relative to. + config = {"standalone_cores": ["kick"], "launchers": [LINK]} + profile = load_emulator_profiles(str(self.emulators))["kick"] + (view, standalone), = platform_layouts("kick", profile, standalone_set(config)) + self.assertTrue(standalone) + self.assertNotIn("standalone_path", view["files"][0]) + + +class UAE4AllOnRetroPie(unittest.TestCase): + """The real profile against the real launcher records.""" + + def test_kickstarts_land_in_amiga(self) -> None: + profile = load_emulator_profiles(str(REPO_ROOT / "emulators"))["uae4all"] + config = load_platform_config("retropie", str(REPO_ROOT / "platforms")) + (view, standalone), = platform_layouts("uae4all", profile, standalone_set(config)) + self.assertTrue(standalone) + for entry in view["files"]: + self.assertEqual( + entry["standalone_path"], "amiga/" + entry["path"].removeprefix("kickstarts/") + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_standalone_links.py b/tests/test_standalone_links.py index 7ea47c50f..1455c7c92 100644 --- a/tests/test_standalone_links.py +++ b/tests/test_standalone_links.py @@ -26,11 +26,16 @@ from common import ( # noqa: E402 load_emulator_profiles, load_platform_config, platform_layouts, + standalone_bindings_for, standalone_destination, - standalone_root_for, standalone_set, ) -from launchers import handed_directories, normalize_handle, standalone_root # noqa: E402 +from launchers import ( # noqa: E402 + bound_path, + handed_directories, + normalize_handle, + standalone_bindings, +) def link(emulator: str, search_dir: str, passes_to: str, **fields: object) -> dict: @@ -101,7 +106,10 @@ class HandedDirectories(unittest.TestCase): class Root(unittest.TestCase): def root(self, *launchers: dict, base: str = "bios") -> str | None: - return standalone_root( + return self.bindings(*launchers, base=base).get("") + + def bindings(self, *launchers: dict, base: str = "bios") -> dict[str, str]: + return standalone_bindings( {"emu"}, PROFILE["standalone_directories"], handed_directories(config(*launchers, base=base)) ) @@ -130,12 +138,15 @@ class Root(unittest.TestCase): "dc", ) - def test_link_of_another_directory_gives_nothing(self) -> None: + def test_link_of_a_subfolder_binds_that_subfolder_only(self) -> None: # A subfolder of the emulator's tree (Dolphin's Triforce) is not the - # directory standalone_path is relative to. - self.assertIsNone(self.root( + # directory standalone_path is relative to, only the files below it move. + found = self.bindings( link("emu", "$bios_path/Triforce", "$XDG_DATA_HOME/emu/Triforce is a symlink to it"), - )) + ) + self.assertEqual(found, {"Triforce": "Triforce"}) + self.assertEqual(bound_path("Triforce/segaboot.gcm", found), "Triforce/segaboot.gcm") + self.assertIsNone(bound_path("boot.bin", found)) def test_two_folders_give_nothing(self) -> None: self.assertIsNone(self.root( @@ -186,13 +197,14 @@ class Layout(unittest.TestCase): union = standalone_set(config(self.LINKED, base="BIOS")) | standalone_set(config( link("emu", "${biosPath}/emu/bios", "symlink ${biosPath}/emu/bios -> ${HOME}/.local/share/emu"), )) - self.assertIsNone(standalone_root_for("emu", PROFILE, union)) + self.assertEqual(standalone_bindings_for("emu", PROFILE, union), {}) def test_destination_function(self) -> None: entry = PROFILE["files"][0] - self.assertEqual(standalone_destination(entry, "dc"), "dc/boot.bin") - self.assertEqual(standalone_destination(entry, None), "boot.bin") - self.assertEqual(standalone_destination(PROFILE["files"][2], "dc"), "dc/shared.bin") + self.assertEqual(standalone_destination(entry, PROFILE, {"": "dc"}), "dc/boot.bin") + self.assertIsNone(standalone_destination(entry, PROFILE, {})) + # The entry both builds share keeps path in a dual-build profile. + self.assertIsNone(standalone_destination(PROFILE["files"][2], PROFILE, {"": "dc"})) class FlycastOnEachPlatform(unittest.TestCase): @@ -215,7 +227,8 @@ class FlycastOnEachPlatform(unittest.TestCase): with self.subTest(platform=platform): cfg = load_platform_config(platform, str(REPO_ROOT / "platforms")) self.assertEqual( - standalone_root_for("flycast", self.profile, standalone_set(cfg)), folder + standalone_bindings_for("flycast", self.profile, standalone_set(cfg)), + {"": folder}, ) def test_no_standalone_file_lands_at_the_bios_root(self) -> None: