diff --git a/emulators/citron.yml b/emulators/citron.yml index d4b5a25ed..8f5fbb6e7 100644 --- a/emulators/citron.yml +++ b/emulators/citron.yml @@ -16,36 +16,43 @@ systems: [nintendo-switch] analysis_date: "2026-03-26" analysis_commit: "79dee42 (depth=1)" mode: standalone +standalone_directories: + - "$XDG_DATA_HOME/citron" + - "$HOME/.local/share/citron" + - "PORTABLE_DIR" + - "CitronPath::KeysDir = keys" # Citron is a community fork of yuzu, continued by Zephyron after yuzu's takedown. -# Key files loaded from /keys/ (fs_paths.h:18, path_util.cpp:124). -# On Linux: $XDG_DATA_HOME/citron/keys/ -# On Batocera: XDG_DATA_HOME=/userdata/saves/switch -> /userdata/saves/switch/citron/keys/ +# Key files loaded from /keys/ (fs_paths.h:18, path_util.cpp:124), +# the folder path_util.cpp generates as CitronPath::KeysDir. The paths below +# are relative to the data folder: $XDG_DATA_HOME/citron on Linux, or the +# user folder (PORTABLE_DIR) beside the executable when it exists. +# On Batocera: 002-adjust-paths.patch sets KeysDir to /userdata/bios/switch. # Firmware NCAs are installed into NAND via UI (main.cpp:1808), not placed as files. # HLE fallbacks exist for fonts, timezone, system version, mii model, ng word lists. files: - name: "prod.keys" required: true - path: "switch/prod.keys" + path: "keys/prod.keys" note: "Production keys for NCA decryption (master, key area, header, titlekek)" source_ref: "src/core/crypto/key_manager.cpp:655-656" - name: "title.keys" required: false - path: "switch/title.keys" + path: "keys/title.keys" note: "Per-title encryption keys (rights_id to titlekey mappings)" source_ref: "src/core/crypto/key_manager.cpp:659-660" - name: "console.keys" required: false - path: "switch/console.keys" + path: "keys/console.keys" note: "Console-specific keys (BIS, SD seed)" source_ref: "src/core/crypto/key_manager.cpp:661-662" - name: "key_retail.bin" required: false - path: "switch/key_retail.bin" + path: "keys/key_retail.bin" size: 160 note: "Amiibo decryption keys (two InternalKey structs, 0x50 bytes each)" source_ref: "src/core/hle/service/nfc/common/amiibo_crypto.cpp:274-292" diff --git a/emulators/shadps4.yml b/emulators/shadps4.yml index 4256ad93e..46db72954 100644 --- a/emulators/shadps4.yml +++ b/emulators/shadps4.yml @@ -17,6 +17,11 @@ cores: systems: - ps4 mode: standalone +standalone_directories: + - "getcwd/user" + - "PORTABLE_DIR" + - "$XDG_DATA_HOME/shadPS4" + - "$HOME/.local/share/shadPS4" analysis_date: "2026-08-10" analysis_commit: "891612c8" @@ -45,28 +50,31 @@ notes: | - font/ from /preinst/common/font/ - font2/ from /system/common/font2/ - Batocera configgen (shadps4Generator.py) configures display, input, and - saves but does not map BIOS/firmware paths. The emulator uses its own - user directory for sys_modules/ and fonts/. + The paths below are relative to the user folder: user/ in the working + directory when it exists, else $XDG_DATA_HOME/shadPS4 or + ~/.local/share/shadPS4 on Linux and %APPDATA%/shadPS4 on Windows + (src/common/path_util.cpp:88-110). sys_modules/ is below it + (path_util.cpp:124) unless sysModulesPath in the config names another + folder (src/core/emulator_settings.cpp:190-195, 535). files: - name: "libSceLibcInternal.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libSceLibcInternal.sprx" + path: "sys_modules/libSceLibcInternal.sprx" description: "PS4 C standard library" source_ref: "src/core/linker.cpp:80-91" - name: "libSceNgs2.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libSceNgs2.sprx" + path: "sys_modules/libSceNgs2.sprx" description: "PS4 next-generation audio system" source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:217" - name: "libSceUlt.sprx" required: false - path: "shadps4/sys_modules/libSceUlt.sprx" + path: "sys_modules/libSceUlt.sprx" description: "PS4 user-level threading library" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:218" @@ -74,13 +82,13 @@ files: - name: "libSceRtc.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libSceRtc.sprx" + path: "sys_modules/libSceRtc.sprx" description: "PS4 real-time clock library" source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:221" - name: "libSceJpegDec.sprx" required: false - path: "shadps4/sys_modules/libSceJpegDec.sprx" + path: "sys_modules/libSceJpegDec.sprx" description: "PS4 JPEG decoder" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:222" @@ -88,83 +96,83 @@ files: - name: "libSceJpegEnc.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libSceJpegEnc.sprx" + path: "sys_modules/libSceJpegEnc.sprx" description: "PS4 JPEG encoder" source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:223" - name: "libScePngEnc.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libScePngEnc.sprx" + path: "sys_modules/libScePngEnc.sprx" description: "PS4 PNG encoder" source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:225" - name: "libSceJson.sprx" required: false - path: "shadps4/sys_modules/libSceJson.sprx" + path: "sys_modules/libSceJson.sprx" description: "PS4 JSON parser" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:226" - name: "libSceJson2.sprx" required: false - path: "shadps4/sys_modules/libSceJson2.sprx" + path: "sys_modules/libSceJson2.sprx" description: "PS4 JSON v2 parser" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:227" - name: "libSceCesCs.sprx" required: false - path: "shadps4/sys_modules/libSceCesCs.sprx" + path: "sys_modules/libSceCesCs.sprx" description: "PS4 character encoding conversion" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:228" - name: "libSceAt9Enc.sprx" required: false - path: "shadps4/sys_modules/libSceAt9Enc.sprx" + path: "sys_modules/libSceAt9Enc.sprx" description: "PS4 ATRAC9 encoder" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:229" - name: "libSceAudiodec.sprx" required: false - path: "shadps4/sys_modules/libSceAudiodec.sprx" + path: "sys_modules/libSceAudiodec.sprx" description: "PS4 audio decoder" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:230" - name: "libSceAudiodecCpu.sprx" required: false - path: "shadps4/sys_modules/libSceAudiodecCpu.sprx" + path: "sys_modules/libSceAudiodecCpu.sprx" description: "PS4 CPU audio decoder" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:231" - name: "libSceAudiodecCpuDdp.sprx" required: false - path: "shadps4/sys_modules/libSceAudiodecCpuDdp.sprx" + path: "sys_modules/libSceAudiodecCpuDdp.sprx" description: "PS4 CPU Dolby Digital Plus decoder" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:232" - name: "libSceAudiodecCpuM4aac.sprx" required: false - path: "shadps4/sys_modules/libSceAudiodecCpuM4aac.sprx" + path: "sys_modules/libSceAudiodecCpuM4aac.sprx" description: "PS4 CPU AAC decoder" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:233" - name: "libSceAudiodecCpuDtsHdLbr.sprx" required: false - path: "shadps4/sys_modules/libSceAudiodecCpuDtsHdLbr.sprx" + path: "sys_modules/libSceAudiodecCpuDtsHdLbr.sprx" description: "PS4 CPU DTS-HD LBR decoder" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:234" - name: "libSceAudiodecCpuHevag.sprx" required: false - path: "shadps4/sys_modules/libSceAudiodecCpuHevag.sprx" + path: "sys_modules/libSceAudiodecCpuHevag.sprx" description: "PS4 CPU HEVAG decoder" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:235" @@ -172,34 +180,34 @@ files: - name: "libSceFont.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libSceFont.sprx" + path: "sys_modules/libSceFont.sprx" description: "PS4 font rendering library" source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:236" - name: "libSceFontFt.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libSceFontFt.sprx" + path: "sys_modules/libSceFontFt.sprx" description: "PS4 FreeType font library" source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:237" - name: "libSceFreeTypeOt.sprx" required: false - path: "shadps4/sys_modules/libSceFreeTypeOt.sprx" + path: "sys_modules/libSceFreeTypeOt.sprx" description: "PS4 FreeType OpenType library" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:238" - name: "libSceFreeTypeOl.sprx" required: false - path: "shadps4/sys_modules/libSceFreeTypeOl.sprx" + path: "sys_modules/libSceFreeTypeOl.sprx" description: "PS4 FreeType outline library" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:239" - name: "libSceFreeTypeOptOl.sprx" required: false - path: "shadps4/sys_modules/libSceFreeTypeOptOl.sprx" + path: "sys_modules/libSceFreeTypeOptOl.sprx" description: "PS4 FreeType optimised outline library" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:240" @@ -207,13 +215,13 @@ files: - name: "libSceRudp.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libSceRudp.sprx" + path: "sys_modules/libSceRudp.sprx" description: "PS4 reliable UDP library" source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:244" - name: "libSceWkFontConfig.sprx" required: false - path: "shadps4/sys_modules/libSceWkFontConfig.sprx" + path: "sys_modules/libSceWkFontConfig.sprx" description: "PS4 WebKit font configuration" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:245" @@ -221,13 +229,13 @@ files: - name: "libSceSystemGesture.sprx" required: false hle_fallback: true - path: "shadps4/sys_modules/libSceSystemGesture.sprx" + path: "sys_modules/libSceSystemGesture.sprx" description: "PS4 system gesture recognition" source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:250" - name: "libSceXml.sprx" required: false - path: "shadps4/sys_modules/libSceXml.sprx" + path: "sys_modules/libSceXml.sprx" description: "PS4 XML parser" note: "No HLE implementation. The module is stubbed when absent." source_ref: "src/core/libraries/sysmodule/sysmodule_internal.cpp:251" diff --git a/emulators/suyu.yml b/emulators/suyu.yml index e95834d14..0f58adbda 100644 --- a/emulators/suyu.yml +++ b/emulators/suyu.yml @@ -16,11 +16,18 @@ systems: [nintendo-switch] analysis_date: "2026-03-26" analysis_commit: "c096d16 (Hengle/suyu-emu mirror, depth=1)" mode: standalone +standalone_directories: + - "$XDG_DATA_HOME/suyu" + - "$HOME/.local/share/suyu" + - "PORTABLE_DIR" + - "SuyuPath::KeysDir = keys" # Suyu is a community fork of yuzu, started after yuzu's legal takedown. -# Key files loaded from /keys/ (fs_paths.h:20, path_util.cpp:124). -# On Linux: $XDG_DATA_HOME/suyu/keys/ -# On Windows: %APPDATA%/suyu/keys/ +# Key files loaded from /keys/ (fs_paths.h:20, path_util.cpp:124), +# the folder path_util.cpp generates as SuyuPath::KeysDir. The paths below +# are relative to the data folder: $XDG_DATA_HOME/suyu on Linux, +# %APPDATA%/suyu on Windows, or the user folder (PORTABLE_DIR) beside the +# executable when it exists. # Firmware NCAs are installed via UI, not placed as files. # HLE fallbacks exist for fonts, timezone, system version, mii model, ng word lists. # Original repos (GitHub + GitLab) are DMCA'd. Analysis from community mirror. @@ -28,25 +35,25 @@ mode: standalone files: - name: "prod.keys" required: true - path: "switch/prod.keys" + path: "keys/prod.keys" note: "Production keys for NCA decryption (master, key area, header, titlekek)" source_ref: "src/core/crypto/key_manager.cpp:654" - name: "title.keys" required: false - path: "switch/title.keys" + path: "keys/title.keys" note: "Per-title encryption keys (rights_id to titlekey mappings)" source_ref: "src/core/crypto/key_manager.cpp:657" - name: "console.keys" required: false - path: "switch/console.keys" + path: "keys/console.keys" note: "Console-specific keys (BIS, SD seed)" source_ref: "src/core/crypto/key_manager.cpp:658" - name: "key_retail.bin" required: false - path: "switch/key_retail.bin" + path: "keys/key_retail.bin" size: 160 note: "Amiibo decryption keys (two InternalKey structs, 0x50 bytes each)" source_ref: "src/core/hle/service/nfc/common/amiibo_crypto.cpp:273-294" diff --git a/install.ps1 b/install.ps1 index de798289b..fcab5282b 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "b9586fece4058b670a4fe83b8c2825b7882024eba816a36353b07a7496af7f1b" + $defaultInstallSha256 = "e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 42f24af07..29335e500 100644 --- a/install.py +++ b/install.py @@ -50,7 +50,7 @@ import weakref import xml.etree.ElementTree as ET import zipfile from collections.abc import Callable, Collection, Iterable, Iterator, Mapping -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePosixPath, PureWindowsPath from typing import Any, BinaryIO, ContextManager, NamedTuple, NoReturn, TextIO # Built as C extensions that a Python compiled without their libraries @@ -3456,8 +3456,19 @@ def _read_limited_json( raise ValueError(f"invalid {label}: {exc}") from exc -def _safe_relative_path(value: object, field: str) -> PurePosixPath: - """Validate a manifest-controlled relative POSIX path.""" +def _path_characters(value: object, field: str, colons: bool = True) -> str: + """The character rules every manifest-controlled relative path meets. + + No backslash, control character, empty segment, trailing slash or drive + prefix, and at most 1024 characters: on Windows a backslash or a C: + segment leaves the folder the path is joined to. A segment that Windows + reads as a drive (C:x, any segment of one letter and a colon in front) + or that ends with a colon is refused wherever it stands, since a later + segment is joined as a drive-relative path. ``colons`` keeps a colon + inside a name: RetroDECK's ares reads bios/snes:st018.data.rom on Linux. + A Windows host refuses it all the same, as an alternate data stream of + the file before the colon. + """ if not isinstance(value, str) or not value or len(value) > 1024: raise ValueError(f"invalid {field}") if ( @@ -3471,12 +3482,79 @@ def _safe_relative_path(value: object, field: str) -> PurePosixPath: # x]0;PWN.bin set the window title in the progress line, # and the file was written under that name. raise ValueError(f"unsafe {field}: {value!r}") - path = PurePosixPath(value) + for part in value.split("/"): + if ":" not in part: + continue + if ( + PureWindowsPath(part).drive or part.endswith(":") + or not colons or os.name == "nt" + ): + raise ValueError(f"unsafe {field}: {value!r}") + return value + + +def _safe_relative_path(value: object, field: str, colons: bool = True) -> PurePosixPath: + """Validate a manifest-controlled relative POSIX path.""" + path = PurePosixPath(_path_characters(value, field, colons)) if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts): raise ValueError(f"unsafe {field}: {value!r}") return path +# The most folders a roms root climbs from the BIOS folder: two, as the rom +# folders files of MinUI, NextUI, MyMinUI (../../Roms) and muOS (../../ROMS) +# record it (platforms/romfolders/*.yml, roms_root.from_bios). +MAX_ROMS_UPS = 2 + + +def _roms_relation(value: object, field: str) -> tuple[int, PurePosixPath]: + """A roms root relative to the BIOS folder: (folders up, names down). + + "." is the BIOS folder itself. Any other value climbs one to + MAX_ROMS_UPS folders, then names plain folders: nothing climbs again + once it went down, and nothing goes down without climbing first. + """ + if value == ".": + return 0, PurePosixPath() + path = PurePosixPath(_path_characters(value, field, colons=False)) + parts = path.parts + ups = next((i for i, part in enumerate(parts) if part != ".."), len(parts)) + names = parts[ups:] + if ( + path.is_absolute() or not 1 <= ups <= MAX_ROMS_UPS + or any(part in ("", ".", "..") for part in names) + ): + raise ValueError(f"unsafe {field}: {value!r}") + return ups, PurePosixPath(*names) + + +def _roms_root(bios_path: Path, from_bios: str) -> Path | None: + """The roms root of a BIOS folder, on the host's own path type. + + The BIOS folder's ancestor at the depth the relation climbs, then the + named folders below it. None when the joined and normalized path is + anything else. + """ + ups, names = _roms_relation(from_bios, "roms_directory.from_bios") + if ups > len(bios_path.parents): + return None + ancestor = bios_path if not ups else bios_path.parents[ups - 1] + expected = ancestor.joinpath(*names.parts) + joined = Path(os.path.normpath(bios_path.joinpath(*PurePosixPath(from_bios).parts))) + if joined != Path(os.path.normpath(expected)) or not _lexically_under(joined, ancestor): + return None + return expected + + +def _lexically_under(candidate: Path, root: Path) -> bool: + """Whether a path built from checked parts stays below *root*.""" + try: + Path(os.path.normpath(candidate)).relative_to(Path(os.path.normpath(root))) + except ValueError: + return False + return True + + def _within(candidate: Path, root: Path) -> bool: """Whether *candidate* stays under *root* once both are resolved.""" try: @@ -3687,10 +3765,11 @@ def _check_roms_directory(data: dict) -> None: return if ( not isinstance(layout, dict) - or set(layout) != {"bios_folder"} + or set(layout) != {"bios_folder", "from_bios"} or len(_safe_relative_path(layout["bios_folder"], "roms_directory.bios_folder").parts) != 1 ): raise ValueError("invalid roms_directory") + _roms_relation(layout["from_bios"], "roms_directory.from_bios") def _check_emulators_directory(data: dict) -> None: @@ -3715,8 +3794,12 @@ def _check_emulators_directory(data: dict) -> None: for template in searched: if not isinstance(template, str) or template.count("{key}") != 1: raise ValueError("invalid emulators_directory.searched") - _safe_relative_path(template.replace("{key}", "key"), "emulators_directory.searched") - _safe_relative_path(layout["config"], "emulators_directory.config") + _safe_relative_path( + template.replace("{key}", "key"), "emulators_directory.searched", colons=False + ) + _safe_relative_path(layout["default"].replace("{key}", "key"), "emulators_directory.default", + colons=False) + _safe_relative_path(layout["config"], "emulators_directory.config", colons=False) def _check_save_directory(data: dict) -> None: @@ -3883,7 +3966,11 @@ def _check_entry( """Check one file entry, return its size and the largest copy it may fetch.""" if not isinstance(entry, dict): raise ValueError(f"{where} must be an object") - dest = str(_safe_relative_path(entry.get("dest"), f"{where}.dest")) + # Below a roms or emulator folder a colon has no use: refused outright. + dest = str(_safe_relative_path( + entry.get("dest"), f"{where}.dest", + colons=entry.get("root") not in ("roms", "emulators"), + )) key = _entry_key({**entry, "dest": dest}) if key in seen_destinations: raise ValueError(f"duplicate manifest destination: {dest}") @@ -5294,17 +5381,22 @@ def place_roms_entries( ) -> tuple[list[dict], list[dict]]: """Find the roms folder for the entries a launcher reads below it. - The platform keeps its BIOS folder in its roms folder under the name the - list gives (ROCKNIX's /storage/roms/bios): a BIOS folder of that name - puts the roms folder above it. Any other is a folder this run cannot - relate to the roms, and the entries are left out with the reason. + The list gives the roms root relative to the BIOS folder, as the OS's + own source places them (ROCKNIX's /storage/roms holds bios/): a BIOS + folder of the platform's name puts the roms root there. Any other is a + folder this run cannot relate to the roms, and the entries are left out + with the reason. """ below = [entry for entry in files if entry.get("root") == "roms"] if not below: return files, [] layout = manifest.get("roms_directory") or {} - if layout and bios_path.name == layout.get("bios_folder"): - _ROMS_DIRS[bios_path] = bios_path.parent + roms = ( + _roms_root(bios_path, layout["from_bios"]) + if layout and bios_path.name == layout.get("bios_folder") else None + ) + if roms is not None: + _ROMS_DIRS[bios_path] = roms return files, [] if say: print(f" {_plural(len(below), 'file')} read from {display}'s roms folder left out: " @@ -5349,7 +5441,11 @@ def place_emulator_entries( return files, [] layout = manifest.get("emulators_directory") or {} home = bios_path.parent - config = home / layout["config"] if layout else None + config = ( + home.joinpath(*_safe_relative_path( + layout["config"], "emulators_directory.config", colons=False).parts) + if layout else None + ) matches = bool(layout) and bios_path.name == layout.get("bios_folder") and config.is_file() moved = _config_keys(config) if matches else set() kept: list[dict] = [] @@ -5358,11 +5454,20 @@ def place_emulator_entries( if entry.get("root") != "emulators": kept.append(entry) continue - key = str(entry["dest"]).split("/", 1)[0] - if matches and key.lower() not in moved: - places = [home / t.replace("{key}", key) for t in layout["searched"]] + key = _safe_relative_path(str(entry["dest"]), "emulators dest", colons=False).parts[0] + places = [ + home.joinpath(*_safe_relative_path( + t.replace("{key}", key), "emulators place", colons=False).parts) + for t in layout.get("searched") or [] + ] + default = home.joinpath( + *_safe_relative_path(layout.get("default", "").replace("{key}", key), + "emulators default", colons=False).parts + ) if matches else None + if (matches and key.lower() not in moved and default is not None + and all(_lexically_under(place, home) for place in [*places, default])): found = next((place for place in places if place.is_dir()), None) - _EMULATOR_DIRS[(bios_path, key)] = found or home / layout["default"].replace("{key}", key) + _EMULATOR_DIRS[(bios_path, key)] = found or default kept.append(entry) continue unplaced.append({ diff --git a/install.sh b/install.sh index 3d4d8ddd0..b5e0bc583 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="b9586fece4058b670a4fe83b8c2825b7882024eba816a36353b07a7496af7f1b" +DEFAULT_INSTALL_SHA256="e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index c72f65bf8..bf070ce79 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1138424, - "sha256": "b9586fece4058b670a4fe83b8c2825b7882024eba816a36353b07a7496af7f1b", + "size": 1142840, + "sha256": "e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1192049, - "sha256": "9da63b607a6ae2a734f4437c224b484229c514b08bbab6341b529ce102de464f" + "size": 1196465, + "sha256": "7ef6e947618ffe51b95823b28ce2a064c6eddf2b7fb77e2066f9c2d279176f03" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1138424, - "sha256": "b9586fece4058b670a4fe83b8c2825b7882024eba816a36353b07a7496af7f1b", + "size": 1142840, + "sha256": "e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1138424, - "sha256": "b9586fece4058b670a4fe83b8c2825b7882024eba816a36353b07a7496af7f1b", + "size": 1142840, + "sha256": "e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c", "executable": false }, { diff --git a/platforms/_registry.yml b/platforms/_registry.yml index bb6e08118..42bf41f68 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -860,13 +860,6 @@ platforms: # sources/RK3566/retroarch.cfg:664-665 at 20261001) and setsettings.sh # leaves it: saves sit beside each game, no single place. save_directory: null - # The BIOS folder sits in the roms directory (/storage/roms/bios, the - # detect entry below), where start_pico8.sh also runs PICO-8 from - # /storage/roms/pico-8. A launcher path below path is placed there, and - # the installer finds that directory as the folder holding bios_folder. - roms_directory: - path: /storage/roms - bios_folder: bios schedule: weekly target_scraper: rocknix_targets target_source: https://github.com/ROCKNIX/distribution/blob/next/projects/ROCKNIX/packages/virtual/emulators/package.mk diff --git a/platforms/batocera.yml b/platforms/batocera.yml index d77026c88..a1702b50f 100644 --- a/platforms/batocera.yml +++ b/platforms/batocera.yml @@ -6615,3 +6615,21 @@ launchers: fails_when_missing: false depends_on_option: neocd_bios source_ref: "package/batocera/core/batocera-configgen/configgen/configgen/generators/libretro/libretroOptions.py:2040-2041" +- emulator: shadps4 + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/system/configs/shadps4 + passes_to: working directory of shadps4Generator.py + fails_when_missing: false + source_ref: "package/batocera/core/batocera-configgen/configgen/configgen/generators/shadps4/shadps4Generator.py:46,\ + \ 228" +- emulator: citron + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/switch + passes_to: "CitronPath::KeysDir in src/common/fs/path_util.cpp returns it, in place\ + \ of citron_path / KEYS_DIR" + fails_when_missing: false + source_ref: "package/batocera/emulators/citron/002-adjust-paths.patch:9-10" diff --git a/platforms/knulli.yml b/platforms/knulli.yml index af3bcc4ef..c8773da9f 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -5685,6 +5685,15 @@ launchers: passes_to: environment FLYCAST_BIOS_PATH fails_when_missing: false source_ref: "package/system/knulli-configgen/configgen/configgen/generators/flycast/flycastGenerator.py:239" +- emulator: shadps4 + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/system/configs/shadps4 + passes_to: working directory of shadps4Generator.py + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/shadps4/shadps4Generator.py:33,\ + \ 164" - emulator: openmsx build: standalone kind: passes_path @@ -6135,3 +6144,20 @@ launchers: depends_on_option: not system.name == 'ti99' source_ref: "package/system/knulli-configgen/configgen/configgen/generators/libretro/libretroMAMEConfig.py:431-432,\ \ 435" +- emulator: suyu + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/saves/switch + passes_to: environment XDG_DATA_HOME + fails_when_missing: false + source_ref: "package/system/knulli-configgen/configgen/configgen/generators/suyu/suyuGenerator.py:41" +- emulator: citron + build: standalone + kind: passes_path + names: [] + search_dir: /userdata/bios/switch + passes_to: "CitronPath::KeysDir in src/common/fs/path_util.cpp returns it, in place\ + \ of citron_path / KEYS_DIR" + fails_when_missing: false + source_ref: "batocera/package/batocera/emulators/citron/002-adjust-paths.patch:9-10" diff --git a/platforms/retrobat.yml b/platforms/retrobat.yml index 4f38a28d7..c6862a448 100644 --- a/platforms/retrobat.yml +++ b/platforms/retrobat.yml @@ -4974,6 +4974,22 @@ launchers: passes_to: exec fails_when_missing: returns no process when the binary is absent source_ref: "emulatorLauncher/Generators/Pico8.Generator.cs:14-18, 56-57" +- emulator: citron + build: standalone + kind: passes_path + names: [] + search_dir: "/user" + passes_to: PORTABLE_DIR, the user folder beside citron.exe + fails_when_missing: false + source_ref: "emulatorLauncher/Generators/Citron.Generator.cs:25, 29-30" +- emulator: shadps4 + build: standalone + kind: passes_path + names: [] + search_dir: "/user" + passes_to: PORTABLE_DIR, the user folder beside shadPS4.exe + fails_when_missing: false + source_ref: "emulatorLauncher/Generators/ShadPS4.Generator.cs:29, 158-160" - emulator: bizhawk build: standalone kind: names_searched diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index c820b6c4f..76d7512f5 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -528,17 +528,23 @@ } }, "roms_directory": { - "description": "Where the platform keeps its roms folder, present when an entry carries root: roms. bios_folder is the name of the BIOS folder inside it: a BIOS folder of that name has the roms folder as its parent, any other leaves root: roms entries out (no_roms_directory).", + "description": "Where the platform keeps its roms root, present when an entry carries root: roms. from_bios is that root relative to the BIOS folder (\".\" for the folder itself, else one or two \"..\" then plain names), as platforms/romfolders/.yml records it from the OS's own source (roms_root.from_bios). It is followed only from a BIOS folder named bios_folder (the platform's base_destination); any other leaves root: roms entries out (no_roms_directory).", "type": "object", "additionalProperties": false, "required": [ - "bios_folder" + "bios_folder", + "from_bios" ], "properties": { "bios_folder": { "type": "string", "minLength": 1, "pattern": "^[^/\\\\]+$" + }, + "from_bios": { + "type": "string", + "pattern": "^(\\.|\\.\\.(/\\.\\.)?(/[^/\\\\:.][^/\\\\:]*)*)$", + "maxLength": 1024 } } }, diff --git a/scripts/common.py b/scripts/common.py index 00f80ac90..aece6b932 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -509,6 +509,14 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") - # The registry key, which per-platform profile fields (installed_on, # not_bundled_on) name: the display name differs (MiSTer FPGA). config["_key"] = platform_name + # Where the platform's games live, as its rom folders file records it + # from the OS's own source (platforms/romfolders/.yml, roms_root). + romfolders = os.path.join(platforms_dir, "romfolders", f"{platform_name}.yml") + if os.path.exists(romfolders): + with open(romfolders) as f: + roms_root = (source_yaml(f) or {}).get("roms_root") + if isinstance(roms_root, dict): + config["_roms_root"] = roms_root _platform_config_cache[cache_key] = config return config @@ -2761,7 +2769,7 @@ def standalone_set(config: dict) -> set[str]: names |= { HANDED_PREFIX + json.dumps( [position, h.emulator, h.folder, h.directory, sorted(h.handles), h.source_ref, - h.roms_folder, sorted(h.systems), h.holds_declared] + h.roms_place, sorted(h.systems), h.holds_declared] ) for position, h in enumerate(handed) } @@ -2808,7 +2816,8 @@ def _handed(standalone_cores: Iterable[str]) -> list: tag[len(HANDED_PREFIX):] ) found.append(((position, tag), HandedDirectory( - emulator, folder, directory, frozenset(handles), ref, roms, frozenset(systems), held + emulator, folder, directory, frozenset(handles), ref, + tuple(roms) if roms else None, frozenset(systems), held, ))) return [h for _key, h in sorted(found, key=lambda pair: pair[0])] @@ -2862,6 +2871,39 @@ EMULATORS_ROOT = "emulators" OUTER_ROOTS = (ROMS_ROOT, EMULATORS_ROOT) +def outer_layout(config: dict, root: str) -> dict | None: + """Where a platform keeps the folder an outer root names, or None. + + roms: the roms root its rom folders file records (``path``, the OS's + spelling, and ``from_bios``, relative to the BIOS folder), with the + BIOS folder's own name, base_destination, that a run's folder must + carry before ``from_bios`` is followed from it. emulators: the + registry's ``emulators_directory``, the folders a launcher searches + for an emulator by key, which no rom folders file describes. + """ + if root == ROMS_ROOT: + found = config.get("_roms_root") or {} + base = str(config.get("base_destination") or "") + path, relation = found.get("path"), found.get("from_bios") + if not (path and relation) or "<" in str(path): + return None + if base: + return {"path": str(path), "from_bios": str(relation), "bios_folder": base} + # A list whose dests start above the BIOS folder (AmberELEC's + # bios/...) installs into the folder holding it: a roms root there + # or below is part of the install tree, written as a plain dest. + declared = declared_root(config) + if not declared: + return None + inside = os.path.normpath(f"{declared}/{relation}") + if inside == ".." or inside.startswith("../"): + return None + return {"path": str(path), "install_prefix": "" if inside == "." else inside} + if root == EMULATORS_ROOT: + return config.get("emulators_directory") or None + return None + + def standalone_layout_path(entry: dict, profile: dict) -> str | None: """The path of an entry relative to the directories its standalone build declares. @@ -2991,6 +3033,11 @@ def platform_layouts( tree, path = placed moved = {**f, "standalone_path": path, "layout_path": standalone_layout_path(f, profile)} + if tree == "install": + # Below the roms root inside the install tree (AmberELEC's + # /storage/roms): a plain dest the platform prefix skips. + files.append({**moved, "unprefixed": True}) + continue # A file the launcher runs from the roms directory (ROCKNIX's # roms/pico-8) or an emulator folder (RetroBat's emulators/pico8) # is relative to it, and says so. diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 4ed76e77a..1f8516755 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -40,6 +40,7 @@ from alternatives import core_file_spares, settle_sets, slot_set_keys from common import resolve_platform_cores as _platform_cores from common import ( OUTER_ROOTS, + outer_layout, installed_data_refs, platform_arg, require_directory, @@ -4686,11 +4687,12 @@ def _record_absent( ) -# What a manifest's roms_directory and emulators_directory carry from the -# registry: the BIOS folder's name, and for emulator folders the places the +# What a manifest's roms_directory and emulators_directory carry: the BIOS +# folder's name, for the roms root its place relative to the BIOS folder +# (romfolders roms_root.from_bios), and for emulator folders the places the # launcher searches, the one a missing folder is created at, and the config # file whose key moves a folder. -MANIFEST_OUTER_FIELDS = ("bios_folder", "searched", "default", "config") +MANIFEST_OUTER_FIELDS = ("bios_folder", "from_bios", "searched", "default", "config") def _record_unplaceable(unplaceable: list[dict], record_omission: Callable[..., None]) -> None: @@ -5259,7 +5261,7 @@ def generate_manifest( if data_directories: result["data_directories"] = data_directories for root in OUTER_ROOTS: - layout = config.get(f"{root}_directory") or {} + layout = outer_layout(config, root) or {} if layout and any(entry.get("root") == root for entry in [*manifest_files, *result["omitted_files"]]): # Where the installer finds the folder: relative to the folder diff --git a/scripts/launchers.py b/scripts/launchers.py index 1f2119d8d..e197636a9 100644 --- a/scripts/launchers.py +++ b/scripts/launchers.py @@ -331,6 +331,9 @@ BIOS_TREE = "" ROMS_TREE = "roms" # The emulator folders of a platform whose registry describes them. EMULATORS_TREE = "emulators" +# The tree a platform's list installs into when it starts above the BIOS +# folder: a plain dest, with no prefix. +INSTALL_TREE = "install" # A folder of neither tree, which no pack and no installer can fill. OUTSIDE_TREE = "outside" Place = tuple[str, str] @@ -382,7 +385,7 @@ class HandedDirectory: directory: str handles: frozenset[str] source_ref: str - roms_folder: str | None = None + roms_place: Place | None = None # The platform systems, by our ids, the record applies to. Empty when # the launcher hands the directory whatever the system. systems: frozenset[str] = frozenset() @@ -393,9 +396,7 @@ class HandedDirectory: """The folder as a place, None when it lies outside both roots.""" if self.folder is not None: return (BIOS_TREE, self.folder) - if self.roms_folder is not None: - return (ROMS_TREE, self.roms_folder) - return None + return self.roms_place # The placeholder a launcher record writes for the system it is launched @@ -491,7 +492,10 @@ def handed_directories(config: dict) -> list[HandedDirectory]: directory=directory, handles=handles, source_ref=str(record.get("source_ref", "")), - roms_folder=None if folder is not None else folder_in_roms(part, config), + roms_place=None if folder is not None else ( + _roms_place(config, folder_in_roms(part, config)) + or _emulators_place(folder_in_emulators(part, config)) + ), systems=systems, holds_declared=bool(folder) and any( d.startswith(f"{folder.lower()}/") for d in declared @@ -500,14 +504,35 @@ def handed_directories(config: dict) -> list[HandedDirectory]: return found +def _emulators_place(path: str | None) -> Place | None: + """A folder an emulator key names, as a place of the emulators tree.""" + return None if path is None else (EMULATORS_TREE, path) + + +def _roms_place(config: dict, path: str | None) -> Place | None: + """A path below the roms root as a place: in the install tree when the + platform installs above its BIOS folder (AmberELEC), else the roms tree.""" + if path is None: + return None + from common import ROMS_ROOT, outer_layout # noqa: PLC0415 + + layout = outer_layout(config, ROMS_ROOT) or {} + if "install_prefix" in layout: + prefix = layout["install_prefix"] + return (INSTALL_TREE, sanitize_pack_path(f"{prefix}/{path}" if prefix else path)) + return (ROMS_TREE, path) + + def folder_in_roms(directory: str, config: dict) -> str | None: """A directory relative to the platform's roms directory, None outside it. - Only a platform whose registry names the roms directory has one - (``roms_directory.path``): ROCKNIX's /storage/roms, which holds its BIOS + Only a platform whose rom folders file records its roms root has one + (``roms_root.path``): ROCKNIX's /storage/roms, which holds its BIOS folder and the folders its launchers run programs from. """ - roms = str((config.get("roms_directory") or {}).get("path") or "").rstrip("/") + from common import ROMS_ROOT, outer_layout # noqa: PLC0415 + + roms = str((outer_layout(config, ROMS_ROOT) or {}).get("path") or "").rstrip("/") directory = directory.strip().rstrip("/") if not roms or not directory.startswith(roms + "/"): return None @@ -526,9 +551,11 @@ def folder_in_emulators(directory: str, config: dict) -> str | None: spelling = str((config.get("emulators_directory") or {}).get("launcher_spelling") or "") if "{key}" not in spelling: return None - pattern = re.escape(spelling).replace(re.escape("{key}"), r"([\w.-]+)") + # The key, then any folder below it: Citron's generator creates + # /user, its portable data folder. + pattern = re.escape(spelling).replace(re.escape("{key}"), r"([\w.-]+)") + r"((?:/[\w.-]+)*)" found = re.fullmatch(pattern, directory.strip()) - return found.group(1) if found else None + return found.group(1) + found.group(2) if found else None @dataclass(frozen=True) @@ -568,7 +595,7 @@ def executed_files(config: dict) -> list[ExecutedFile]: f"{directories[0]}/{name}" if directories[0] else name )) elif roms is not None: - place = (ROMS_TREE, sanitize_pack_path(f"{roms}/{name}" if roms else name)) + place = _roms_place(config, sanitize_pack_path(f"{roms}/{name}" if roms else name)) elif emulator_folder is not None: place = (EMULATORS_TREE, sanitize_pack_path(f"{emulator_folder}/{name}")) else: @@ -618,8 +645,9 @@ def _record_places( up = _below(base, handle) if inner else None if up: found.append(("", (target[0], _join(target[1], up)))) - if not found and record.roms_folder is not None: - found.append(("", (ROMS_TREE, _join(record.roms_folder, inner)))) + if not found and record.roms_place is not None: + root, folder = record.roms_place + found.append(("", (root, _join(folder, inner)))) return found @@ -627,14 +655,19 @@ def _bind( records: list[HandedDirectory], links: list[HandedDirectory], wanted: set[str], + named: Mapping[str, str], pools: Mapping[str, str], ) -> dict[str, set[Place]]: """The places each subfolder of the layout is given by ``records``. A record whose handle is a declared directory, or lies below one, gives - that subfolder. One whose handle lies above a declared directory gives - the declared directory where it lands: RetroPie configures XRoar with - --prefix=$md_inst, so ${prefix}/share/xroar/roms is $md_inst/share/xroar/roms. + that subfolder, and a handle ``named`` gives the one folder it stands + for. One whose handle lies above a declared directory, or is the + environment variable the declared directory is spelled from + ($XDG_DATA_HOME/suyu), gives the declared directory where it lands: + RetroPie configures XRoar with --prefix=$md_inst, so + ${prefix}/share/xroar/roms is $md_inst/share/xroar/roms. When it lands + in no folder a pack fills, the layout is read outside the trees. ``links`` are the records followed from a directory outside the trees. A handle in ``pools`` names a list of directories the program searches @@ -653,16 +686,27 @@ def _bind( (record.holds_declared, place) ) continue + if handle in named: + for rest, place in _record_places(record, "", links): + found.setdefault(_join(named[handle], rest), set()).add(place) for directory in wanted: sub = _below(handle, directory) - inner = "" - if sub is None: - up = _below(directory, handle) - if not up: - continue - sub, inner = "", up - for rest, place in _record_places(record, inner, links): - found.setdefault(_join(sub, rest), set()).add(place) + if sub is not None: + for rest, place in _record_places(record, "", links): + found.setdefault(_join(sub, rest), set()).add(place) + continue + if handle == directory: + continue + inner = next( + (up for spelled in (f"${handle}", handle) + if (up := _below(directory, spelled)) is not None), + None, + ) + if inner is None: + continue + places = _record_places(record, inner, links) + for rest, place in places or [("", (OUTSIDE_TREE, ""))]: + found.setdefault(rest, set()).add(place) for sub, entries in pooled.items(): # The directory the platform's own list fills, else the first of the # BIOS tree, else the first a pack fills. @@ -714,17 +758,28 @@ def standalone_bindings( """ names = set(names) pools = {normalize_handle(k): str(v).strip("/") for k, v in (pools or {}).items()} - wanted = {normalize_handle(d) for d in declared} - set(pools) - if not wanted and not pools: + # " = " names the code's handle for one folder of the + # layout: Citron's CitronPath::KeysDir is its keys/ folder, and a patch + # that rewrites the handle moves that folder alone. + named: dict[str, str] = {} + wanted: set[str] = set() + for raw in declared: + handle, sep, sub = str(raw).partition(" = ") + if sep: + named[normalize_handle(handle)] = sanitize_pack_path(sub.strip()) + else: + wanted.add(normalize_handle(handle)) + wanted -= set(pools) + if not wanted and not named and not pools: return {} systems = set(systems) records = [h for h in handed if h.emulator in names] generic = [h for h in records if not h.systems] specific = [h for h in records if h.systems & systems] - found = _bind(generic, generic, wanted, pools) + found = _bind(generic, generic, wanted, named, pools) # A link is followed whatever system it was written for. - found.update(_bind(specific, specific + generic, wanted, pools)) + found.update(_bind(specific, specific + generic, wanted, named, pools)) if EXECUTABLE_DIRECTORY in wanted: paths = list(layout) for run in executed: diff --git a/scripts/packextras.py b/scripts/packextras.py index d665a5e4f..16a5d64b3 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -18,7 +18,7 @@ from common import resolve_local_file from common import resolve_platform_cores from common import core_system_dirs, platform_layouts, system_dir_path from common import standalone_set as platform_standalone_set -from common import OUTER_ROOTS, sanitize_pack_path +from common import OUTER_ROOTS, outer_layout, sanitize_pack_path from common import size_fits from validation import deliverable, read_from_save_dir, read_from_system_dir from packpaths import libretro_systems, save_destination, save_layout @@ -44,6 +44,10 @@ def extra_full_destination(fe: dict, prefix: str) -> str: """ if fe.get("root") == "save": return str(fe.get("pack_path") or "") + if fe.get("unprefixed"): + # A path of the install tree outside the BIOS folder (AmberELEC's + # pico-8/ beside bios/): no platform prefix in front. + return sanitize_pack_path(fe.get("destination", fe.get("name", ""))) if fe.get("root") in OUTER_ROOTS: return outer_destination( fe["root"], sanitize_pack_path(fe.get("destination", fe.get("name", ""))) @@ -573,7 +577,7 @@ def _agnostic_scan_extras( # The scan walks the BIOS tree: a file read below the save # directory is placed by the gap report's pass. if (read_from_save_dir(f, standalone) or f.get("unplaced") - or f.get("root") in OUTER_ROOTS): + or f.get("root") in OUTER_ROOTS or f.get("unprefixed")): continue # An agnostic BIOS mode says the BIOS filename is free, not that # every file the emulator loads is. A free filename still has a @@ -1024,12 +1028,10 @@ class ExtraPlacement: self.native_systems = native_slug_systems(config) self.flat = bool(config.get("firmware_folder_flat")) self.save = save_layout(config) - # The roms directory and emulator folders the registry describes, + # The roms root and emulator folders the platform's files describe, # for a file a launcher runs from them (ROCKNIX's roms/pico-8, # RetroBat's emulators/pico8). - self.outer = { - root for root in OUTER_ROOTS if config.get(f"{root}_directory") - } + self.outer = {root for root in OUTER_ROOTS if outer_layout(config, root)} self.config = config self.norm_map = ( {_norm_system_id(sid): sid for sid in self.platform_systems} @@ -1050,6 +1052,8 @@ class ExtraPlacement: """ if entry.get("unplaced"): return None + if entry.get("unprefixed"): + return dest, dest if entry.get("root") in OUTER_ROOTS: root = entry["root"] return (dest, outer_destination(root, dest)) if root in self.outer else None @@ -1339,7 +1343,7 @@ def _collect_emulator_extras( # covered elsewhere belongs to another program's file. # Nor is one placed outside the BIOS tree by the layout. if (fname not in covered_names or f.get("unsourceable") or f.get("unplaced") - or f.get("root") in OUTER_ROOTS): + or f.get("root") in OUTER_ROOTS or f.get("unprefixed")): continue # Installed with the emulator or by the platform's package, # as the first pass already decided for the same entry. @@ -1586,6 +1590,8 @@ def _report_extra(u: dict, name: str, dest: str) -> dict: extra["system_dir"] = u["system_dir"] if u.get("layout_path"): extra["layout_path"] = u["layout_path"] + if u.get("unprefixed"): + extra["unprefixed"] = True # The archive layout the core opens ranks same-named copies by what # they hold. Without it the first copy indexed won. if u.get("contents"): diff --git a/scripts/scraper/batocera_launchers.py b/scripts/scraper/batocera_launchers.py index 4a675866d..aac64eb9a 100644 --- a/scripts/scraper/batocera_launchers.py +++ b/scripts/scraper/batocera_launchers.py @@ -105,9 +105,10 @@ def _path_value(node: ast.AST | None, constants: dict[str, str]) -> str | None: and isinstance(node.func, ast.Name) and node.func.id == "Path" and len(node.args) == 1 - and isinstance(node.args[0], ast.Constant) ): - return str(node.args[0].value) + if isinstance(node.args[0], ast.Constant): + return str(node.args[0].value) + return _path_value(node.args[0], constants) if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): left = _path_value(node.left, constants) right = node.right.value if isinstance(node.right, ast.Constant) else None @@ -954,6 +955,44 @@ def config_directories( return launchers +# Generators that start the emulator from a folder they enter with os.chdir. +WORKING_DIRECTORY_GENERATORS = ("shadps4.shadps4Generator",) + + +def working_directories(cg: Configgen) -> list[Launcher]: + """os.chdir() in the generators above, before the command runs. + + shadPS4 reads its user folder below the working directory first + (std::filesystem::current_path() / "user"): Batocera enters + /userdata/system/configs/shadps4. + """ + launchers = [] + for module in WORKING_DIRECTORY_GENERATORS: + src = cg.generator(module) + calls = _calls(src.tree, attr="chdir") + if len(calls) != 1 or len(calls[0].args) != 1: + src.fail("one os.chdir(folder) call") + call = calls[0] + argument = call.args[0] + constants = dict(cg.paths) + spans = [(call.lineno, call.end_lineno or call.lineno)] + if isinstance(argument, ast.Name): + scope = _innermost_function(src, call) + assigned = src.assignment(argument.id, scope) + spans.append((assigned.lineno, assigned.end_lineno or assigned.lineno)) + folder = _path_value(assigned.value, constants) + else: + folder = _path_value(argument, constants) + if folder is None: + src.fail("os.chdir names no folder") + launchers.append(Launcher( + emulator=cg.emulator(module), build="standalone", kind="passes_path", names=[], + search_dir=folder, passes_to=f"working directory of {src.path.rsplit('/', 1)[-1]}", + source_ref=src.ref(spans), + )) + return launchers + + # libretro core options whose value names the BIOS file the core opens. FILE_OPTIONS = (("_o2em_options", "o2em_bios"), ("_neocd_options", "neocd_bios")) @@ -1006,14 +1045,64 @@ def core_file_options(cg: Configgen) -> list[Launcher]: return launchers +# Batocera's patches to an emulator's own path code. Each rewrites one of +# the folders the emulator derives (yuzu's family generates every user +# folder through GeneratePath) to a fixed folder of /userdata. +PATH_PATCHES = ("package/batocera/emulators/citron/002-adjust-paths.patch",) +_PATH_REMOVED = re.compile(r"^-\s*Generate\w+Path\((\w+Path::\w+),\s*(.+?)\);\s*$") +_PATH_ADDED = re.compile(r'^\+\s*Generate\w+Path\((\w+Path::\w+),\s*"(/userdata/bios[^"]*)"\);\s*$') + + +def patched_paths(read: Callable[[str], str], patches: tuple[str, ...] = PATH_PATCHES) -> list[Launcher]: + """The emulator folders a path patch points at the BIOS tree. + + A patch line replacing `GenerateCitronPath(CitronPath::KeysDir, + citron_path / KEYS_DIR)` with a literal /userdata/bios folder hands + that folder to the emulator under the handle the code names it by. + The emulator is the package the patch belongs to. + """ + launchers: list[Launcher] = [] + for path in patches: + lines = read(path).splitlines() + target = "" + removed: dict[str, tuple[int, str]] = {} + found = 0 + for number, line in enumerate(lines, 1): + if line.startswith("+++ b/"): + target = line[len("+++ b/"):] + continue + if (old := _PATH_REMOVED.match(line)): + removed[old.group(1)] = (number, old.group(2)) + continue + if not (new := _PATH_ADDED.match(line)): + continue + handle, folder = new.groups() + if handle not in removed or not target: + raise LauncherShapeError(f"{path}:{number}: {handle} added with no line it replaces") + first, original = removed[handle] + found += 1 + launchers.append(Launcher( + emulator=path.split("/")[-2], build="standalone", kind="passes_path", names=[], + search_dir=folder, + passes_to=f"{handle} in {target} returns it, in place of {original}", + fails_when_missing=False, + source_ref=f"{path}:{first}-{number}", + )) + if not found: + raise LauncherShapeError(f"{path}: no path rewritten to /userdata/bios") + return launchers + + EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = ( duckstation, duckstation_legacy, hatari, xemu, melonds, mupen64plus, x16emu, gsplus, lexaloffle, environment_directories, config_directories, - core_file_options, + core_file_options, working_directories, ) def extract_launchers(read: Callable[[str], str]) -> list[dict[str, Any]]: """Every launcher record, read through `read(repo-relative path)`.""" cg = Configgen(read) - return [launcher.entry() for extractor in EXTRACTORS for launcher in extractor(cg)] + launchers = [launcher for extractor in EXTRACTORS for launcher in extractor(cg)] + launchers += patched_paths(read) + return [launcher.entry() for launcher in launchers] diff --git a/scripts/scraper/knulli_launchers.py b/scripts/scraper/knulli_launchers.py index c25dfb55a..b9916f0c8 100644 --- a/scripts/scraper/knulli_launchers.py +++ b/scripts/scraper/knulli_launchers.py @@ -30,6 +30,7 @@ import yaml try: from .batocera_launchers import ( + PATH_PATCHES, Configgen, config_directories, _path_value, @@ -41,11 +42,14 @@ try: hatari, melonds, mupen64plus, + patched_paths, x16emu, + working_directories, xemu, ) from .configgen_launchers import ( Launcher, + _calls, LauncherShapeError, Source, _assigned_name, @@ -53,6 +57,7 @@ try: from .shell_launchers import LauncherPatternMissing, Script, source_ref except ImportError: from batocera_launchers import ( # type: ignore[no-redef] + PATH_PATCHES, Configgen, config_directories, _path_value, @@ -64,11 +69,14 @@ except ImportError: hatari, melonds, mupen64plus, + patched_paths, x16emu, + working_directories, xemu, ) from configgen_launchers import ( # type: ignore[no-redef] Launcher, + _calls, LauncherShapeError, Source, _assigned_name, @@ -85,7 +93,6 @@ LIBRETRO_OPTIONS = f"{GENERATORS}/libretro/libretroOptions.py" PICO8_INSTALLER = "board/fsoverlay/usr/bin/knulli-install-pico8" ES_FEATURES = "package/emulationstation/knulli-es-system/es_features.yml" -# Batocera's extractors that read KNULLI's generators unchanged. # The generators of Batocera's config_directories that KNULLI keeps in the # same shape: openMSX's filepool lines and amiberry's rom_path. CONFIG_DIRECTORY_GENERATORS = ("openmsx.openmsxGenerator", "amiberry.amiberryGenerator") @@ -95,9 +102,10 @@ def knulli_config_directories(cg: Configgen) -> list[Launcher]: return config_directories(cg, CONFIG_DIRECTORY_GENERATORS) +# Batocera's extractors that read KNULLI's generators unchanged. SHARED = ( duckstation, duckstation_legacy, hatari, xemu, melonds, mupen64plus, x16emu, - environment_directories, knulli_config_directories, + environment_directories, working_directories, knulli_config_directories, ) @@ -1063,14 +1071,65 @@ EXTRACTORS: tuple[Callable[[Configgen], list[Launcher]], ...] = ( ) -def extract_launchers(read: Callable[[str], str]) -> list[dict[str, Any]]: - """Every launcher record, read through `read(repo-relative path)`.""" +# Generators whose command sets the emulator's XDG data folder, the parent +# of every user folder a yuzu-family emulator derives. +DATA_HOME_GENERATORS = ("suyu.suyuGenerator",) + + +def data_homes(cg: Configgen) -> list[Launcher]: + """Command(..., env={"XDG_DATA_HOME": }) in the generators above. + + The path need not lie in /userdata/bios: KNULLI's suyu keeps its data + in /userdata/saves/switch, which tells where its keys are not. + """ + launchers = [] + for module in DATA_HOME_GENERATORS: + src = cg.generator(module) + found = [ + (_path_value(item, cg.paths), key) + for call in _calls(src.tree, attr="Command") + for keyword in call.keywords + if keyword.arg == "env" and isinstance(keyword.value, ast.Dict) + for key, item in zip(keyword.value.keys, keyword.value.values) + if isinstance(key, ast.Constant) and key.value == "XDG_DATA_HOME" + ] + if not found or found[0][0] is None: + src.fail("the command environment sets no XDG_DATA_HOME path") + value, node = found[0] + launchers.append(Launcher( + emulator=cg.emulator(module), build="standalone", kind="passes_path", names=[], + search_dir=value, passes_to="environment XDG_DATA_HOME", + source_ref=src.ref([(node.lineno, node.end_lineno)]), + )) + return launchers + + +# KNULLI builds Batocera's emulator packages from this submodule, at the +# batocera.linux commit it pins: Citron's path patch is there. +BATOCERA_SUBMODULE = "batocera" + + +def extract_launchers( + read: Callable[[str], str], batocera: Callable[[str], str] | None = None +) -> list[dict[str, Any]]: + """Every launcher record, read through `read(repo-relative path)`. + + `batocera` reads the batocera submodule at its pinned commit. The + records it gives name the file by its path in KNULLI's repository. + """ cg = configgen(read) launchers = [launcher for extractor in EXTRACTORS for launcher in extractor(cg)] launchers += pico8_installer(read, cg.bios) launchers += boot_disks(cg, read) for spec in GENERIC_SITES: launchers += bios_sites(cg, spec) + launchers += data_homes(cg) + if batocera is not None: + prefix = f"{BATOCERA_SUBMODULE}/" + launchers += patched_paths( + lambda path: batocera(path.removeprefix(prefix)), + tuple(prefix + patch for patch in PATH_PATCHES), + ) return [launcher.entry() for launcher in launchers] diff --git a/scripts/scraper/knulli_scraper.py b/scripts/scraper/knulli_scraper.py index 47af8e50c..abae8d987 100644 --- a/scripts/scraper/knulli_scraper.py +++ b/scripts/scraper/knulli_scraper.py @@ -45,12 +45,15 @@ from .batocera_scraper import ( requirements_from_systems, ) from .es_emulators import attach_system_cores -from .knulli_launchers import extract_launchers +from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers PLATFORM_NAME = "knulli" REPO = "knulli-cfw/knulli-linux" _RAW = "https://raw.githubusercontent.com/knulli-cfw/knulli-linux/{ref}/" +# The batocera.linux revision KNULLI's submodule pins, and its files. +_SUBMODULE = "https://api.github.com/repos/knulli-cfw/knulli-linux/contents/{path}?ref={ref}" +_BATOCERA_RAW = "https://raw.githubusercontent.com/batocera-linux/batocera.linux/{ref}/" SOURCE_TEMPLATE = _RAW + "package/system/knulli-scripts/scripts/knulli-systems" CONFIGGEN_DEFAULTS_TEMPLATE = ( _RAW + "package/system/knulli-configgen/configs/configgen-defaults.yml" @@ -83,6 +86,23 @@ class Scraper(BaseScraper): except urllib.error.URLError as e: raise ConnectionError(f"Failed to fetch {url}: {e}") from e + def submodule_revision(self) -> str: + """The batocera.linux commit KNULLI's batocera submodule pins at the tag.""" + import json # noqa: PLC0415 + + from .base_scraper import github_headers # noqa: PLC0415 + + url = _SUBMODULE.format(path=BATOCERA_SUBMODULE, ref=self.tag) + req = urllib.request.Request(url, headers=github_headers()) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + found = json.loads(resp.read().decode("utf-8")) + except urllib.error.URLError as e: + raise ConnectionError(f"Failed to fetch {url}: {e}") from e + if not isinstance(found, dict) or found.get("type") != "submodule" or not found.get("sha"): + raise RuntimeError(f"{url}: {BATOCERA_SUBMODULE} is not a submodule") + return str(found["sha"]) + def validate_format(self, raw_data: str) -> bool: has_dict = re.search(r"^systems\s*=\s*\{", raw_data, re.MULTILINE) is not None return has_dict and "def checkBios" in raw_data and '"biosFiles"' in raw_data @@ -126,8 +146,10 @@ class Scraper(BaseScraper): # configgen hands a libretro core /userdata/saves/, as # Batocera's does: the systems each core runs for are its save folders. result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped) + revision = self.submodule_revision() result["launchers"] = extract_launchers( - lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path) + lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path), + lambda path: self._fetch_text(_BATOCERA_RAW.format(ref=revision) + path), ) return result diff --git a/scripts/scraper/retrobat_launchers.py b/scripts/scraper/retrobat_launchers.py index b1c960b51..4b8b393ea 100644 --- a/scripts/scraper/retrobat_launchers.py +++ b/scripts/scraper/retrobat_launchers.py @@ -25,6 +25,7 @@ SOURCE_FILES = ( f"{GENERATORS}/Bizhawk.CoreOptions.cs", f"{GENERATORS}/Bizhawk.Generator.cs", f"{GENERATORS}/Blastem.Generator.cs", + f"{GENERATORS}/Citron.Generator.cs", f"{GENERATORS}/Demul.Generator.cs", f"{GENERATORS}/Desmume.Generator.cs", f"{GENERATORS}/Dolphin.Generator.cs", @@ -58,6 +59,7 @@ SOURCE_FILES = ( f"{GENERATORS}/Rpcs3.Generator.cs", f"{GENERATORS}/SSF.Generator.cs", f"{GENERATORS}/ScummVm.Generator.cs", + f"{GENERATORS}/ShadPS4.Generator.cs", f"{GENERATORS}/Snes9x.Generator.cs", f"{GENERATORS}/Tsugaru.Generator.cs", f"{GENERATORS}/Uae.Generator.cs", @@ -3038,6 +3040,48 @@ def _per_system(src: Source, record: Record) -> list[Record]: return out +# Generators that create a portable user folder beside the emulator's binary, +# and the emulator each one starts: the folder AppConfig.GetFullPath gives. +PORTABLE_USERS = ( + ("Citron.Generator.cs", "citron", "citron.exe"), + ("ShadPS4.Generator.cs", "shadps4", "shadPS4.exe"), +) +_EMULATOR_FOLDER = ( + r"GetEmulatorPathAndExe\(emulator, out string path, out string exe, \"[^\"]+\"\)" + r"|string path = AppConfig\.GetFullPath\(emulator\);" +) + + +def portable_users(sources: dict[str, Source]) -> list[Record]: + """The portable user folder a generator creates beside the binary. + + The emulator folder comes from AppConfig.GetFullPath(emulator), and the + generator creates /user before launching: the emulator then + keeps its data there (keys/, sys_modules/) in place of %APPDATA%. + """ + records = [] + for name, emulator, binary in PORTABLE_USERS: + src = sources[f"{GENERATORS}/{name}"] + found = src.find(_EMULATOR_FOLDER, f"{emulator} folder") + user = src.find(r'string userFolder = Path\.Combine\(path, "(\w+)"\);', "user folder") + create = src.find( + r"Directory\.CreateDirectory\(userFolder\)", "user folder creation", user.end() + ) + records.append(_record( + emulator, + "standalone", + "passes_path", + [], + source_ref((src.path, [ + src.lines(found.start(), found.end()), src.lines(user.start(), create.end()), + ])), + search_dir=f'/{user.group(1)}', + passes_to=f"PORTABLE_DIR, the user folder beside {binary}", + fails_when_missing=False, + )) + return records + + def pico8(sources: dict[str, Source]) -> list[Record]: """The PICO-8 binary the generator runs, from an emulator folder. @@ -3345,6 +3389,7 @@ EXTRACTORS: tuple[Callable[[dict[str, Source]], list[Record]], ...] = ( pcsx2, pcsx2_libretro, pico8, + portable_users, psx_cascades, psxmame, raine, diff --git a/scripts/verify.py b/scripts/verify.py index 9d703a646..7a05a1be9 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -794,6 +794,8 @@ def find_undeclared_files( entry["members"] = members if f.get("layout_path"): entry["layout_path"] = f["layout_path"] + if f.get("unprefixed"): + entry["unprefixed"] = True if f.get("unplaced"): entry["unplaced"] = f["unplaced"] elif f.get("root") in OUTER_ROOTS: diff --git a/tests/test_install.py b/tests/test_install.py index 262a033ed..9ca17cabb 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1135,6 +1135,103 @@ class StandaloneCopiesReadWhereThePackPutTheFile(unittest.TestCase): install._validate_manifest(manifest, "emudeck") +class OuterRootsAreUntrusted(unittest.TestCase): + """roms_directory and emulators_directory come from a downloaded list: each + path in them meets the rules a BIOS destination meets, and each folder + they resolve to stays where the platform's layout puts it.""" + + ROMS = {"bios_folder": "bios", "from_bios": ".."} + EMULATORS = { + "bios_folder": "bios", "searched": ["frontend/{key}", "emulators/{key}"], + "default": "emulators/{key}", "config": "frontend/launcher.cfg", + } + UNSAFE = ("..\\x", "C:", "C:/x", "..//x", "../", "../x\x01", "x" * 1025) + + def test_unsafe_from_bios_is_refused(self): + for value in (*self.UNSAFE, "../../../x", "../x/..", "x", "./x", "/x", "..x", None): + with self.subTest(from_bios=value), self.assertRaises(ValueError): + install._check_roms_directory({"roms_directory": {**self.ROMS, "from_bios": value}}) + + def test_the_relations_the_rom_folders_files_use_are_accepted(self): + bios = Path("/card/system/bios") + expected = { + ".": bios, "..": Path("/card/system"), "../roms": Path("/card/system/roms"), + "../Roms": Path("/card/system/Roms"), "../../Roms": Path("/card/Roms"), + "../../ROMS": Path("/card/ROMS"), + } + for value, root in expected.items(): + with self.subTest(from_bios=value): + install._check_roms_directory({"roms_directory": {**self.ROMS, "from_bios": value}}) + self.assertEqual(install._roms_root(bios, value), root) + self.assertEqual(install.MAX_ROMS_UPS, 2) + + def test_a_list_without_a_roms_root_leaves_the_files_out(self): + files = [{"dest": "pico-8/pico8_64", "root": "roms", "cores": ["x"]}] + kept, left = install.place_roms_entries(files, Path("/card/roms/bios"), {}, "x", say=False) + self.assertEqual((kept, [o["reason"] for o in left]), ([], ["no_roms_directory"])) + + def test_unsafe_emulator_folders_are_refused(self): + for field in ("searched", "default", "config"): + for value in self.UNSAFE + ("../{key}", "/abs/{key}"): + layout = dict(self.EMULATORS) + if field == "searched": + layout["searched"] = [value if "{key}" in value else value + "/{key}"] + layout["default"] = layout["searched"][0] + elif field == "default": + layout["default"] = value + else: + layout["config"] = value + with self.subTest(field=field, value=value), self.assertRaises(ValueError): + install._check_emulators_directory({"emulators_directory": layout}) + + def test_a_drive_or_colon_segment_is_refused_on_either_host(self): + from pathlib import PureWindowsPath + + bios = PureWindowsPath("C:/RetroBat/bios") + for host in ("posix", "nt"): + with unittest.mock.patch.object(install.os, "name", host): + for value in ("../C:x", "..\\x", "../x:", "../Roms:x"): + with self.subTest(host=host, from_bios=value), self.assertRaises(ValueError): + install._check_roms_directory( + {"roms_directory": {**self.ROMS, "from_bios": value}}) + for value in ("C:x/{key}", "x:/{key}", "emu:x/{key}"): + layout = {**self.EMULATORS, "searched": [value], "default": value} + with self.subTest(host=host, searched=value), self.assertRaises(ValueError): + install._check_emulators_directory({"emulators_directory": layout}) + for dest in ("bios/C:x.bin", "bios/x:", "a:b/c.bin"): + with self.subTest(host=host, dest=dest), self.assertRaises(ValueError): + install._safe_relative_path(dest, "dest") + with self.subTest(host=host), self.assertRaises(ValueError): + install._safe_relative_path("pico8/x:y", "dest", colons=False) + # Each of these leaves the folder on Windows once joined. + self.assertEqual(PureWindowsPath(bios, "..", "D:x"), PureWindowsPath("D:x")) + self.assertEqual(PureWindowsPath("C:x").drive, "C:") + # RetroDECK's ares names are kept where the host is not Windows. + with unittest.mock.patch.object(install.os, "name", "posix"): + install._safe_relative_path("bios/snes:st018.data.rom", "dest") + with unittest.mock.patch.object(install.os, "name", "nt"), self.assertRaises(ValueError): + install._safe_relative_path("bios/snes:st018.data.rom", "dest") + + def test_an_emulator_folder_stays_below_the_platform_folder(self): + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp) + (home / "bios").mkdir() + (home / "frontend").mkdir() + (home / "frontend" / "launcher.cfg").write_text("") + manifest = {"emulators_directory": self.EMULATORS} + for dest in ("../x/prog", "C:/prog", "prog\\..\\..\\x/a"): + with self.subTest(dest=dest), self.assertRaises(ValueError): + install.place_emulator_entries( + [{"dest": dest, "root": "emulators"}], home / "bios", manifest, "x", + say=False) + kept, left = install.place_emulator_entries( + [{"dest": "prog/prog.exe", "root": "emulators"}], home / "bios", manifest, "x", + say=False) + self.assertEqual((len(kept), left), (1, [])) + folder, rest = install._entry_place(home / "bios", kept[0]) + self.assertEqual((folder, rest), (home / "emulators" / "prog", "prog.exe")) + + class DetectionFollowsTheRegistry(unittest.TestCase): """The registry declared RetroPie at ~/RetroPie/BIOS and BizHawk at ~/.config/BizHawk/Firmware, every manifest published it, and the diff --git a/tests/test_launcher_paths.py b/tests/test_launcher_paths.py new file mode 100644 index 000000000..2f0cb8f1e --- /dev/null +++ b/tests/test_launcher_paths.py @@ -0,0 +1,217 @@ +"""Folders a platform's launcher hands an emulator, read from its code. + +Batocera patches Citron's path code to read its keys from /userdata/bios/ +switch and starts shadPS4 from /userdata/system/configs/shadps4, KNULLI sets +suyu's XDG data folder, and RetroBat creates a portable user folder beside +Citron and shadPS4. Each fixture is a cut-down copy of the lines at the +revisions the platform files pin, and a binding reads the record the +extractor writes. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import sys +import textwrap +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from launchers import handed_directories, standalone_bindings # noqa: E402 +from scraper import knulli_launchers as kl # noqa: E402 +from scraper import retrobat_launchers as rl # noqa: E402 +from scraper.batocera_launchers import ( # noqa: E402 + GENERATORS, + IMPORTER, + PATHS_MODULE, + Configgen, + LauncherShapeError, + patched_paths, + working_directories, +) + +PATCH = "package/batocera/emulators/citron/002-adjust-paths.patch" +PATCH_TEXT = textwrap.dedent("""\ + diff --git a/src/common/fs/path_util.cpp b/src/common/fs/path_util.cpp + index f92cbe0f7b..9e3dcd9c18 100644 + --- a/src/common/fs/path_util.cpp + +++ b/src/common/fs/path_util.cpp + @@ -121,12 +121,12 @@ public: + GenerateCitronPath(CitronPath::ConfigDir, citron_path_config); + GenerateCitronPath(CitronPath::DumpDir, citron_path / DUMP_DIR); + - GenerateCitronPath(CitronPath::KeysDir, citron_path / KEYS_DIR); + + GenerateCitronPath(CitronPath::KeysDir, "/userdata/bios/switch"); + - GenerateCitronPath(CitronPath::LogDir, citron_path / LOG_DIR); + + GenerateCitronPath(CitronPath::LogDir, "/userdata/system/logs"); + """) + +PATHS = """from pathlib import Path +from typing import Final + +USERDATA: Final = Path('/userdata') +BIOS: Final = USERDATA / 'bios' +SAVES: Final = USERDATA / 'saves' +CONFIGS: Final = USERDATA / 'system' / 'configs' +""" +IMPORTER_TEXT = """_LEGACY_GENERATOR_MAP = {} +_GENERATOR_MAP = {} +""" +SHADPS4 = f"{GENERATORS}/shadps4/shadps4Generator.py" +SHADPS4_TEXT = textwrap.dedent("""\ + import os + from ...batoceraPaths import CONFIGS + + class ShadPS4Generator: + def generate(self, system, rom): + configPath = CONFIGS / "shadps4" + userConfigPath = configPath / "user" + os.chdir(configPath) + return None + """) + + +def records(entries: list) -> list[dict]: + return [entry.entry() if hasattr(entry, "entry") else entry for entry in entries] + + +class APatchRewritingAFolder(unittest.TestCase): + def test_the_bios_folder_a_patch_gives_is_a_record(self): + (record,) = records(patched_paths({PATCH: PATCH_TEXT}.__getitem__)) + self.assertEqual( + (record["emulator"], record["kind"], record["search_dir"], record["source_ref"]), + ("citron", "passes_path", "/userdata/bios/switch", f"{PATCH}:8-9"), + ) + self.assertIn("CitronPath::KeysDir", record["passes_to"]) + self.assertIn("citron_path / KEYS_DIR", record["passes_to"]) + + def test_a_patch_that_no_longer_rewrites_a_bios_folder_stops_the_scrape(self): + text = PATCH_TEXT.replace('"/userdata/bios/switch"', '"/userdata/saves/switch"') + with self.assertRaises(LauncherShapeError): + patched_paths({PATCH: text}.__getitem__) + + def test_the_named_handle_binds_its_folder_alone(self): + config = {"base_destination": "bios", "standalone_cores": ["citron"], "launchers": + records(patched_paths({PATCH: PATCH_TEXT}.__getitem__))} + found = standalone_bindings( + {"citron"}, ["$XDG_DATA_HOME/citron", "CitronPath::KeysDir = keys"], + handed_directories(config), + ) + self.assertEqual(found, {"keys": ("", "switch")}) + + +class AFolderTheGeneratorEnters(unittest.TestCase): + def tree(self, text: str = SHADPS4_TEXT) -> Configgen: + files = {PATHS_MODULE: PATHS, IMPORTER: IMPORTER_TEXT, SHADPS4: text} + return Configgen(files.__getitem__) + + def test_the_working_directory_is_a_record(self): + (record,) = records(working_directories(self.tree())) + self.assertEqual( + (record["emulator"], record["search_dir"], record["passes_to"], record["source_ref"]), + ("shadps4", "/userdata/system/configs/shadps4", + "working directory of shadps4Generator.py", f"{SHADPS4}:6, 8"), + ) + + def test_a_generator_that_no_longer_enters_a_folder_stops_the_scrape(self): + with self.assertRaises(LauncherShapeError): + working_directories(self.tree(SHADPS4_TEXT.replace("os.chdir(configPath)", "pass"))) + + def test_the_user_folder_below_it_is_outside_the_bios_tree(self): + config = {"base_destination": "bios", "standalone_cores": ["shadps4"], + "launchers": records(working_directories(self.tree()))} + found = standalone_bindings( + {"shadps4"}, ["getcwd/user", "$XDG_DATA_HOME/shadPS4"], handed_directories(config) + ) + self.assertEqual(found, {"": ("outside", "")}) + + +class ADataFolderTheEnvironmentSets(unittest.TestCase): + SUYU = f"{kl.GENERATORS}/suyu/suyuGenerator.py" + TEXT = textwrap.dedent("""\ + from ...batoceraPaths import CACHE, CONFIGS, SAVES + + class SuyuGenerator: + def generate(self, system, rom): + commandArray = ["/usr/bin/suyu", "-f", "-g", rom] + return Command.Command(array=commandArray, env={ + "XDG_CONFIG_HOME": CONFIGS, + "XDG_DATA_HOME": SAVES / "switch", + }) + """) + + def test_the_data_folder_is_a_record_and_binds_outside(self): + files = {f"{kl.CONFIGGEN}/batoceraPaths.py": PATHS, + f"{kl.GENERATORS}/importer.py": IMPORTER_TEXT, self.SUYU: self.TEXT} + (record,) = records(kl.data_homes(kl.configgen(files.__getitem__))) + self.assertEqual( + (record["emulator"], record["search_dir"], record["passes_to"], record["source_ref"]), + ("suyu", "/userdata/saves/switch", "environment XDG_DATA_HOME", f"{self.SUYU}:8"), + ) + config = {"base_destination": "bios", "standalone_cores": ["suyu"], "launchers": [record]} + found = standalone_bindings( + {"suyu"}, ["$XDG_DATA_HOME/suyu", "SuyuPath::KeysDir = keys"], + handed_directories(config), + ) + self.assertEqual(found, {"": ("outside", "")}) + + +class APortableUserFolder(unittest.TestCase): + CITRON = """ + public override System.Diagnostics.ProcessStartInfo Generate(string system, string emulator, string core, string rom, string playersControllers, ScreenResolution resolution) + { + if (!GetEmulatorPathAndExe(emulator, out string path, out string exe, "citron.exe")) + return null; + + // Ensure user folder exists + string userFolder = Path.Combine(path, "user"); + if (!Directory.Exists(userFolder)) try { Directory.CreateDirectory(userFolder); } + catch { } + } + """ + SHADPS4 = """ + public override System.Diagnostics.ProcessStartInfo Generate(string system, string emulator, string core, string rom, string playersControllers, ScreenResolution resolution) + { + string path = AppConfig.GetFullPath(emulator); + } + + private void SetupConfigurationJSON(string path, string rom) + { + string userFolder = Path.Combine(path, "user"); + if (!Directory.Exists(userFolder)) + try { Directory.CreateDirectory(userFolder); } catch { } + } + """ + + def sources(self) -> dict[str, rl.Source]: + return { + f"{rl.GENERATORS}/Citron.Generator.cs": rl.Source( + "c", textwrap.dedent(self.CITRON).lstrip("\n")), + f"{rl.GENERATORS}/ShadPS4.Generator.cs": rl.Source( + "s", textwrap.dedent(self.SHADPS4).lstrip("\n")), + } + + def test_each_generator_gives_its_user_folder(self): + found = rl.portable_users(self.sources()) + self.assertEqual( + [(r["emulator"], r["search_dir"], r["source_ref"]) for r in found], + [("citron", '/user', "c:3, 7-8"), + ("shadps4", '/user', "s:3, 8-10")], + ) + + def test_the_user_folder_binds_below_the_emulator_folder(self): + config = { + "base_destination": "bios", "standalone_cores": ["citron"], + "emulators_directory": {"launcher_spelling": ''}, + "launchers": rl.portable_users(self.sources())[:1], + } + found = standalone_bindings( + {"citron"}, ["$XDG_DATA_HOME/citron", "PORTABLE_DIR"], handed_directories(config) + ) + self.assertEqual(found, {"": ("emulators", "citron/user")}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_launcher_placement.py b/tests/test_launcher_placement.py index 50cafe33c..2e23f5c75 100644 --- a/tests/test_launcher_placement.py +++ b/tests/test_launcher_placement.py @@ -274,13 +274,36 @@ class AProgramRunFromItsFolder(unittest.TestCase): })) (self.platforms / "_registry.yml").write_text(yaml.safe_dump({"platforms": { **{name: {"config": f"{name}.yml"} for name in launchers}, - "romsplat": {"config": "romsplat.yml", - "roms_directory": {"path": "/storage/roms", "bios_folder": "bios"}}, "emuplat": {"config": "emuplat.yml", "emulators_directory": { "launcher_spelling": '', "bios_folder": "bios", "searched": ["frontend/{key}", "emulators/{key}"], "default": "emulators/{key}", "config": "frontend/launcher.cfg"}}, }})) + # The roms root as the OS's own source records it (romfolders). + (self.platforms / "romfolders").mkdir() + (self.platforms / "romfolders" / "romsplat.yml").write_text(yaml.safe_dump({ + "setup": "romsplat", "platform": "romsplat", + "roms_root": {"path": "/storage/roms", "from_bios": "..", "source_ref": "os x.mk:1"}, + "folders": [], + })) + # A list that installs into the roms root itself, its BIOS folder + # one of the folders there (AmberELEC's /storage/roms). + (self.platforms / "rootplat.yml").write_text(yaml.safe_dump({ + "platform": "rootplat", "version": "1", "verification_mode": "existence", + "base_destination": "", "cores": ["prog"], "standalone_cores": ["prog"], + "launchers": [{**run, "names": ["prog"], "search_dir": "/storage/roms/prog"}], + "systems": {"prog-system": {"files": [ + {"name": "own.bin", "destination": "bios/own.bin", "required": False}, + ]}}, + })) + registry = yaml.safe_load((self.platforms / "_registry.yml").read_text()) + registry["platforms"]["rootplat"] = {"config": "rootplat.yml"} + (self.platforms / "_registry.yml").write_text(yaml.safe_dump(registry)) + (self.platforms / "romfolders" / "rootplat.yml").write_text(yaml.safe_dump({ + "setup": "rootplat", "platform": "rootplat", + "roms_root": {"path": "/storage/roms", "from_bios": "..", "source_ref": "os x.mk:1"}, + "folders": [], + })) (self.emulators / "prog.yml").write_text(yaml.safe_dump({ "emulator": "Prog", "type": "standalone", "cores": ["prog"], "systems": ["prog-system"], @@ -332,11 +355,20 @@ class AProgramRunFromItsFolder(unittest.TestCase): sorted((f["dest"], f.get("root")) for f in manifest["files"]), [("prog/prog", "roms"), ("prog/prog.dat", "roms")], ) - self.assertEqual(manifest["roms_directory"], {"bios_folder": "bios"}) + self.assertEqual(manifest["roms_directory"], {"bios_folder": "bios", "from_bios": ".."}) entries = self.verify("romsplat") self.assertEqual(sorted(u["path"] for u in entries), ["prog/prog", "prog/prog.dat"]) self.assertTrue(all(u["in_pack"] is False for u in entries)) + def test_a_roms_root_inside_the_install_tree_is_a_plain_dest(self) -> None: + self.assertEqual(self.pack("rootplat"), ["prog/prog", "prog/prog.dat"]) + manifest = self.manifest("rootplat") + self.assertEqual( + sorted((f["dest"], f.get("root")) for f in manifest["files"] if "prog" in f["dest"]), + [("prog/prog", None), ("prog/prog.dat", None)], + ) + self.assertNotIn("roms_directory", manifest) + def test_run_from_an_emulator_folder_is_named(self) -> None: self.assertEqual(self.pack("exeplat"), []) manifest = self.manifest("exeplat") @@ -367,6 +399,19 @@ class AProgramRunFromItsFolder(unittest.TestCase): self.assertEqual({o["reason"] for o in left}, {"no_roms_directory"}) with self.assertRaises(ValueError): install._check_roms_directory({"files": files}) + # A roms root beside the BIOS folder (Batocera's ../roms). + beside = {**manifest, "roms_directory": {"bios_folder": "bios", "from_bios": "../roms"}} + install._check_roms_directory(beside) + disk = self.root / "userdata" / "bios" + install.place_roms_entries(files, disk, beside, "romsplat", say=False) + self.assertEqual(install._entry_place(disk, kept_entry := files[0])[0], + self.root / "userdata" / "roms") + self.assertEqual(kept_entry["root"], "roms") + for climbing in ("roms", "../roms/..", "/storage/roms"): + with self.subTest(from_bios=climbing), self.assertRaises(ValueError): + install._check_roms_directory( + {**manifest, "roms_directory": {"bios_folder": "bios", "from_bios": climbing}} + ) def test_run_from_an_emulator_folder_the_registry_describes(self) -> None: self.assertEqual(self.pack("emuplat"), []) diff --git a/tests/test_launchers.py b/tests/test_launchers.py index 59cb86c6b..c211f63ca 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -595,8 +595,13 @@ class PlatformRecords(unittest.TestCase): self.assertEqual(sorted(named - cores), []) def test_every_reference_is_in_configgen(self): + # Or in a patch Batocera applies to an emulator's own path code. + from scraper.batocera_launchers import PATH_PATCHES + for record in self.config["launchers"]: - self.assertTrue(record["source_ref"].startswith(f"{CONFIGGEN}/"), record) + self.assertTrue( + record["source_ref"].startswith((f"{CONFIGGEN}/", *PATH_PATCHES)), record + ) def test_the_schema_describes_what_the_scraper_writes(self): fields = {f.name for f in dataclasses.fields(Launcher)} diff --git a/tests/test_launchers_knulli.py b/tests/test_launchers_knulli.py index 62062446a..2657c8dd5 100644 --- a/tests/test_launchers_knulli.py +++ b/tests/test_launchers_knulli.py @@ -623,8 +623,12 @@ class PlatformFile(unittest.TestCase): def test_every_reference_is_in_knulli_code(self): for record in self.config["launchers"]: + # Or in Batocera's patches, read from KNULLI's batocera submodule. self.assertTrue( - record["source_ref"].startswith((f"{kl.CONFIGGEN}/", kl.PICO8_INSTALLER)), record + record["source_ref"].startswith(( + f"{kl.CONFIGGEN}/", kl.PICO8_INSTALLER, + *(f"{kl.BATOCERA_SUBMODULE}/{patch}" for patch in kl.PATH_PATCHES), + )), record ) def test_every_emulator_is_one_knulli_names(self):