From 7e22c3811369375b1062ad386ab14a6ac6012704 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:53:49 +0100 Subject: [PATCH] feat: place shadps4 files below the configs root Batocera and KNULLI start shadPS4 from CONFIGS/shadps4, so its user folder sits below /userdata/system/configs. The scrapers read that root from batoceraPaths and write configs_root with its source_ref. The manifest names it configs_directory, and install.py honours it through the same checks as roms_directory: character rules, at most two ups, the ancestor match and the lexical bound. --- install.ps1 | 2 +- install.py | 131 ++++++++++++++------------ install.sh | 2 +- install/launchers/index.json | 16 ++-- platforms/batocera.yml | 4 + platforms/knulli.yml | 4 + schemas/install-manifest.schema.json | 39 ++++++-- schemas/platform.schema.json | 24 +++++ scripts/common.py | 12 ++- scripts/launchers.py | 31 +++++- scripts/scraper/batocera_launchers.py | 26 +++++ scripts/scraper/batocera_scraper.py | 5 +- scripts/scraper/knulli_scraper.py | 5 + scripts/verify.py | 1 + tests/test_install.py | 10 +- tests/test_launcher_paths.py | 40 ++++++++ tests/test_launcher_placement.py | 14 +-- 17 files changed, 275 insertions(+), 91 deletions(-) diff --git a/install.ps1 b/install.ps1 index fcab5282b..615aac053 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c" + $defaultInstallSha256 = "fc575b6b4ac26a12d6d2b28aaa5ffe7dcfbbd57ead80e719e2ce57b8af00c6a8" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 29335e500..f44329b0d 100644 --- a/install.py +++ b/install.py @@ -143,6 +143,7 @@ OMISSION_REASONS = { "no_save_system": "no system runs the core", "outside_bios_folder": "read outside the BIOS folder", "no_roms_directory": "no roms folder", + "no_configs_directory": "no configs folder", "no_emulators_directory": "no emulator folder", } # What a file is when it is not firmware, as the manifest's category field @@ -3528,14 +3529,20 @@ def _roms_relation(value: object, field: str) -> tuple[int, PurePosixPath]: return ups, PurePosixPath(*names) -def _roms_root(bios_path: Path, from_bios: str) -> Path | None: - """The roms root of a BIOS folder, on the host's own path type. +# The roots a list places files below by their place relative to the BIOS +# folder: the roms root (ROCKNIX's /storage/roms) and the frontend's +# configs folder (Batocera's /userdata/system/configs). +RELATIVE_ROOTS = ("roms", "configs") + + +def _relative_root(bios_path: Path, from_bios: str, root: str = "roms") -> Path | None: + """A relative root of a BIOS folder, on the host's own path type. The BIOS folder's ancestor at the depth the relation climbs, then the named folders below it. None when the joined and normalized path is anything else. """ - ups, names = _roms_relation(from_bios, "roms_directory.from_bios") + ups, names = _roms_relation(from_bios, f"{root}_directory.from_bios") if ups > len(bios_path.parents): return None ancestor = bios_path if not ups else bios_path.parents[ups - 1] @@ -3671,8 +3678,9 @@ class SavePlace(NamedTuple): # BIOS folder -> where the files read below its frontend's save directory go, # set once the save directory is known (save_directory_for). _SAVE_DIRS: dict[Path, SavePlace] = {} -# BIOS folder -> the platform's roms folder, set by place_roms_entries. -_ROMS_DIRS: dict[Path, Path] = {} +# (BIOS folder, root) -> the platform's roms or configs folder, set by +# place_relative_entries. +_RELATIVE_DIRS: dict[tuple[Path, str], Path] = {} # (BIOS folder, emulator key) -> the emulator folder, set by # place_emulator_entries. _EMULATOR_DIRS: dict[tuple[Path, str], Path] = {} @@ -3686,11 +3694,11 @@ def _entry_place(root: Path, entry: dict) -> tuple[Path, object]: core's own folder where the frontend sorts saves by core. Any other is relative to the BIOS folder. """ - if entry.get("root") == "roms": - roms = _ROMS_DIRS.get(root) - if roms is None: - raise ValueError(f"no roms folder known for {entry.get('dest')}") - return roms, entry.get("dest") + if entry.get("root") in RELATIVE_ROOTS: + folder = _RELATIVE_DIRS.get((root, entry["root"])) + if folder is None: + raise ValueError(f"no {entry['root']} folder known for {entry.get('dest')}") + return folder, entry.get("dest") if entry.get("root") == "emulators": key, _, rest = str(entry.get("dest")).partition("/") folder = _EMULATOR_DIRS.get((root, key)) @@ -3731,7 +3739,7 @@ def _entry_key(entry: dict) -> str: if root == "emulator": # A path in one emulator's own folder: two emulators read two files. return f"emulator\0{','.join(entry.get('cores') or [])}\0{dest}" - return f"{root}\0{dest}" if root in ("save", "roms", "emulators") else dest + return f"{root}\0{dest}" if root in ("save", "emulators", *RELATIVE_ROOTS) else dest def _check_root(entry: dict, dest: str, omitted: bool = False) -> None: @@ -3739,7 +3747,8 @@ def _check_root(entry: dict, dest: str, omitted: bool = False) -> None: the emulator's own folder, outside the BIOS tree), and core_folder one folder name of a save entry.""" root = entry.get("root") - if root not in (None, "save", "roms", "emulators") and not (omitted and root == "emulator"): + if root not in (None, "save", "emulators", *RELATIVE_ROOTS) and not ( + omitted and root == "emulator"): raise ValueError(f"invalid root for {dest}") if root == "emulators" and len(PurePosixPath(dest).parts) < 2: raise ValueError(f"an emulators entry names its folder first: {dest}") @@ -3752,24 +3761,27 @@ def _check_root(entry: dict, dest: str, omitted: bool = False) -> None: raise ValueError(f"core_folder must be one folder name for {dest}") -def _check_roms_directory(data: dict) -> None: - """The manifest's roms_directory, which every installable root: roms entry needs.""" - layout = data.get("roms_directory") +def _check_relative_directories(data: dict) -> None: + """The manifest's roms_directory and configs_directory, which every + installable root: roms or root: configs entry needs.""" installable = [ entry for field in ("files", "case_variants") for entry in data.get(field) or [] if isinstance(entry, dict) ] - if layout is None: - if any(entry.get("root") == "roms" for entry in installable): - raise ValueError("entries below the roms folder and no roms_directory") - return - if ( - not isinstance(layout, dict) - or set(layout) != {"bios_folder", "from_bios"} - or len(_safe_relative_path(layout["bios_folder"], "roms_directory.bios_folder").parts) != 1 - ): - raise ValueError("invalid roms_directory") - _roms_relation(layout["from_bios"], "roms_directory.from_bios") + for root in RELATIVE_ROOTS: + field = f"{root}_directory" + layout = data.get(field) + if layout is None: + if any(entry.get("root") == root for entry in installable): + raise ValueError(f"entries below the {root} folder and no {field}") + continue + if ( + not isinstance(layout, dict) + or set(layout) != {"bios_folder", "from_bios"} + or len(_safe_relative_path(layout["bios_folder"], f"{field}.bios_folder").parts) != 1 + ): + raise ValueError(f"invalid {field}") + _roms_relation(layout["from_bios"], f"{field}.from_bios") def _check_emulators_directory(data: dict) -> None: @@ -3969,7 +3981,7 @@ def _check_entry( # Below a roms or emulator folder a colon has no use: refused outright. dest = str(_safe_relative_path( entry.get("dest"), f"{where}.dest", - colons=entry.get("root") not in ("roms", "emulators"), + colons=entry.get("root") not in ("emulators", *RELATIVE_ROOTS), )) key = _entry_key({**entry, "dest": dest}) if key in seen_destinations: @@ -4109,7 +4121,7 @@ def _validate_manifest(data: object, plat: str) -> dict: for index, entry in enumerate(data_dirs): _check_data_directory(entry, f"data_directories[{index}]") _check_save_directory(data) - _check_roms_directory(data) + _check_relative_directories(data) _check_emulators_directory(data) declared_total_files = data.get("total_files") @@ -5376,39 +5388,41 @@ def place_save_entries( return kept, unplaced -def place_roms_entries( +def place_relative_entries( files: list[dict], bios_path: Path, manifest: dict, display: str, say: bool = True, ) -> tuple[list[dict], list[dict]]: - """Find the roms folder for the entries a launcher reads below it. + """Find the roms and configs folders for the entries read below them. - The list gives the roms root relative to the BIOS folder, as the OS's - own source places them (ROCKNIX's /storage/roms holds bios/): a BIOS - folder of the platform's name puts the roms root there. Any other is a - folder this run cannot relate to the roms, and the entries are left out - with the reason. + The list gives each root relative to the BIOS folder, as the OS's own + source places them (ROCKNIX's /storage/roms holds bios/, Batocera's + configs folder is ../system/configs): a BIOS folder of the platform's + name puts the root there. Any other is a folder this run cannot relate + to the root, and the entries are left out with the reason. """ - below = [entry for entry in files if entry.get("root") == "roms"] - if not below: - return files, [] - layout = manifest.get("roms_directory") or {} - roms = ( - _roms_root(bios_path, layout["from_bios"]) - if layout and bios_path.name == layout.get("bios_folder") else None - ) - if roms is not None: - _ROMS_DIRS[bios_path] = roms - return files, [] - if say: - print(f" {_plural(len(below), 'file')} read from {display}'s roms folder left out: " - f"{bios_path} is not its {layout.get('bios_folder') or 'BIOS'} folder.") - kept = [entry for entry in files if entry.get("root") != "roms"] - unplaced = [ - {"dest": entry["dest"], "name": PurePosixPath(entry["dest"]).name, - "system": "", "systems": list(entry.get("systems") or []), "required": False, - "reason": "no_roms_directory", "cores": entry.get("cores"), "root": "roms"} - for entry in below - ] - return kept, unplaced + unplaced: list[dict] = [] + for root in RELATIVE_ROOTS: + below = [entry for entry in files if entry.get("root") == root] + if not below: + continue + layout = manifest.get(f"{root}_directory") or {} + folder = ( + _relative_root(bios_path, layout["from_bios"], root) + if layout and bios_path.name == layout.get("bios_folder") else None + ) + if folder is not None: + _RELATIVE_DIRS[(bios_path, root)] = folder + continue + if say: + print(f" {_plural(len(below), 'file')} read from {display}'s {root} folder left " + f"out: {bios_path} is not its {layout.get('bios_folder') or 'BIOS'} folder.") + files = [entry for entry in files if entry.get("root") != root] + unplaced += [ + {"dest": entry["dest"], "name": PurePosixPath(entry["dest"]).name, + "system": "", "systems": list(entry.get("systems") or []), "required": False, + "reason": f"no_{root}_directory", "cores": entry.get("cores"), "root": root} + for entry in below + ] + return files, unplaced def _config_keys(path: Path) -> set[str]: @@ -16098,7 +16112,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: display = display_name(plat_name, manifest) files, unplaced = place_save_entries( files, bios_path, manifest, args.save_dir, display, say=False) - files, beyond = place_roms_entries(files, bios_path, manifest, display, say=not listing) + files, beyond = place_relative_entries( + files, bios_path, manifest, display, say=not listing) files, elsewhere = place_emulator_entries( files, bios_path, manifest, display, say=not listing) unplaced = [*unplaced, *beyond, *elsewhere] diff --git a/install.sh b/install.sh index b5e0bc583..acd8fb1d8 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c" +DEFAULT_INSTALL_SHA256="fc575b6b4ac26a12d6d2b28aaa5ffe7dcfbbd57ead80e719e2ce57b8af00c6a8" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index bf070ce79..119df24a0 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1142840, - "sha256": "e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c", + "size": 1143647, + "sha256": "fc575b6b4ac26a12d6d2b28aaa5ffe7dcfbbd57ead80e719e2ce57b8af00c6a8", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1196465, - "sha256": "7ef6e947618ffe51b95823b28ce2a064c6eddf2b7fb77e2066f9c2d279176f03" + "size": 1197272, + "sha256": "d37af192ab41abfd9e2f0d82c67fcc4ca3252e864a91582b4f6b23f840bc00aa" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1142840, - "sha256": "e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c", + "size": 1143647, + "sha256": "fc575b6b4ac26a12d6d2b28aaa5ffe7dcfbbd57ead80e719e2ce57b8af00c6a8", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1142840, - "sha256": "e076488b9766904d7f859e0463e55214fed9386dcaaca362b04a371a820c6c7c", + "size": 1143647, + "sha256": "fc575b6b4ac26a12d6d2b28aaa5ffe7dcfbbd57ead80e719e2ce57b8af00c6a8", "executable": false }, { diff --git a/platforms/batocera.yml b/platforms/batocera.yml index 968db3d35..312f75f09 100644 --- a/platforms/batocera.yml +++ b/platforms/batocera.yml @@ -6598,3 +6598,7 @@ launchers: \ / KEYS_DIR" fails_when_missing: false source_ref: "package/batocera/emulators/citron/002-adjust-paths.patch:9-10" +configs_root: + path: /userdata/system/configs + from_bios: ../system/configs + source_ref: "package/batocera/core/batocera-configgen/configgen/configgen/batoceraPaths.py:14, 19-20, 25" diff --git a/platforms/knulli.yml b/platforms/knulli.yml index 846328981..51edf4c42 100644 --- a/platforms/knulli.yml +++ b/platforms/knulli.yml @@ -6150,3 +6150,7 @@ launchers: \ / KEYS_DIR" fails_when_missing: false source_ref: "batocera/package/batocera/emulators/citron/002-adjust-paths.patch:9-10" +configs_root: + path: /userdata/system/configs + from_bios: ../system/configs + source_ref: "package/system/knulli-configgen/configgen/configgen/batoceraPaths.py:14, 19-20, 25" diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index 76d7512f5..9c3da6fe5 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -207,7 +207,8 @@ "no_save_system", "outside_bios_folder", "no_roms_directory", - "no_emulators_directory" + "no_emulators_directory", + "no_configs_directory" ], "description": "Why the file is left out, as a code. enumDescriptions gives the meaning of each code, in the order of enum.", "enumDescriptions": [ @@ -222,7 +223,8 @@ "The launcher gives each system its own save folder and no system of the platform runs the core.", "The platform runs the emulator from a folder outside the BIOS tree and links none of the BIOS tree where the emulator reads this file (root: emulator).", "The platform's launcher runs the file from its roms folder (root: roms) and the run cannot tell where that folder is: the BIOS folder is not the one the platform keeps in it.", - "The platform's launcher runs the file from one of its emulator folders (root: emulators) and the run cannot place it there: the BIOS folder is not the one beside the platform's launcher config, or that config sets the folder." + "The platform's launcher runs the file from one of its emulator folders (root: emulators) and the run cannot place it there: the BIOS folder is not the one beside the platform's launcher config, or that config sets the folder.", + "The platform's launcher starts the emulator from a folder of its configs folder (root: configs) and the run cannot tell where that folder is: the BIOS folder is not the one the platform keeps beside it." ] }, "category": { @@ -367,12 +369,13 @@ } }, "root": { - "description": "save: dest is relative to the save directory the frontend hands the core (RETRO_ENVIRONMENT_GET_SAVE_DIRECTORY), not to the BIOS directory. emulator: dest is the path the emulator reads in its own folder, which the platform keeps outside the BIOS tree (reason outside_bios_folder). roms: dest is relative to the platform's roms folder (roms_directory). emulators: dest is the emulator folder's key then the path in it (emulators_directory). Absent for a file of the BIOS tree.", + "description": "save: dest is relative to the save directory the frontend hands the core (RETRO_ENVIRONMENT_GET_SAVE_DIRECTORY), not to the BIOS directory. emulator: dest is the path the emulator reads in its own folder, which the platform keeps outside the BIOS tree (reason outside_bios_folder). roms: dest is relative to the platform's roms folder (roms_directory). emulators: dest is the emulator folder's key then the path in it (emulators_directory). configs: dest is relative to the frontend's configs folder (configs_directory). Absent for a file of the BIOS tree.", "enum": [ "save", "emulator", "roms", - "emulators" + "emulators", + "configs" ] }, "core_folder": { @@ -581,6 +584,27 @@ "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$" } } + }, + "configs_directory": { + "description": "Where the frontend keeps its emulators' configs, present when an entry carries root: configs. from_bios is that folder relative to the BIOS folder, as the platform's own path code builds it (Batocera's batoceraPaths CONFIGS, ../system/configs), with the rules of roms_directory.from_bios. It is followed only from a BIOS folder named bios_folder; any other leaves root: configs entries out (no_configs_directory).", + "type": "object", + "additionalProperties": false, + "required": [ + "bios_folder", + "from_bios" + ], + "properties": { + "bios_folder": { + "type": "string", + "minLength": 1, + "pattern": "^[^/\\\\]+$" + }, + "from_bios": { + "type": "string", + "pattern": "^(\\.|\\.\\.(/\\.\\.)?(/[^/\\\\:.][^/\\\\:]*)*)$", + "maxLength": 1024 + } + } } }, "$defs": { @@ -599,7 +623,7 @@ "dest": { "type": "string", "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).+$", - "description": "Path below base_destination, below the save directory when root is save, below the roms folder when root is roms, or in an emulator folder when root is emulators (its key first)." + "description": "Path below base_destination, below the save directory when root is save, below the roms folder when root is roms, below the configs folder when root is configs, or in an emulator folder when root is emulators (its key first)." }, "sha1": { "type": "string", @@ -1082,11 +1106,12 @@ } }, "root": { - "description": "save: dest is relative to the save directory the frontend hands the core (RETRO_ENVIRONMENT_GET_SAVE_DIRECTORY), not to the BIOS directory. roms: dest is relative to the platform's roms folder, which roms_directory locates (a file its launcher runs from there, ROCKNIX's roms/pico-8). emulators: dest is the key of an emulator folder of the platform, then the path in it, located by emulators_directory (RetroBat's emulators/pico8). Absent for a file of the BIOS tree.", + "description": "save: dest is relative to the save directory the frontend hands the core (RETRO_ENVIRONMENT_GET_SAVE_DIRECTORY), not to the BIOS directory. roms: dest is relative to the platform's roms folder, which roms_directory locates (a file its launcher runs from there, ROCKNIX's roms/pico-8). emulators: dest is the key of an emulator folder of the platform, then the path in it, located by emulators_directory (RetroBat's emulators/pico8). configs: dest is relative to the frontend's configs folder, which configs_directory locates (Batocera starts shadPS4 from configs/shadps4). Absent for a file of the BIOS tree.", "enum": [ "save", "roms", - "emulators" + "emulators", + "configs" ] }, "core_folder": { diff --git a/schemas/platform.schema.json b/schemas/platform.schema.json index 0d409de16..1bed464e1 100644 --- a/schemas/platform.schema.json +++ b/schemas/platform.schema.json @@ -188,6 +188,30 @@ } } } + }, + "configs_root": { + "description": "Where the frontend keeps its emulators' configs, as its own path code builds it (Batocera's batoceraPaths CONFIGS): path as the OS spells it, from_bios relative to the BIOS folder. A launcher that starts an emulator from a folder below it (shadPS4's configs/shadps4) places that emulator's user folder there, and the install manifest writes such a file as root: configs.", + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "from_bios", + "source_ref" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "from_bios": { + "type": "string", + "minLength": 1 + }, + "source_ref": { + "type": "string", + "minLength": 1 + } + } } }, "$defs": { diff --git a/scripts/common.py b/scripts/common.py index 9522a8618..8f58c4708 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -2825,7 +2825,8 @@ OUTSIDE_BIOS_FOLDER = "outside_bios_folder" # describes. Each is located by the registry entry _directory. ROMS_ROOT = "roms" EMULATORS_ROOT = "emulators" -OUTER_ROOTS = (ROMS_ROOT, EMULATORS_ROOT) +CONFIGS_ROOT = "configs" +OUTER_ROOTS = (ROMS_ROOT, EMULATORS_ROOT, CONFIGS_ROOT) def outer_layout(config: dict, root: str) -> dict | None: @@ -2858,6 +2859,15 @@ def outer_layout(config: dict, root: str) -> dict | None: return {"path": str(path), "install_prefix": "" if inside == "." else inside} if root == EMULATORS_ROOT: return config.get("emulators_directory") or None + if root == CONFIGS_ROOT: + # The frontend's configs folder, as its path code builds it + # (configs_root, Batocera's batoceraPaths CONFIGS). + found = config.get("configs_root") or {} + base = str(config.get("base_destination") or "") + if not (found.get("path") and found.get("from_bios") and base): + return None + return {"path": str(found["path"]), "from_bios": str(found["from_bios"]), + "bios_folder": base} return None diff --git a/scripts/launchers.py b/scripts/launchers.py index b79cb5678..aabac759c 100644 --- a/scripts/launchers.py +++ b/scripts/launchers.py @@ -331,6 +331,8 @@ BIOS_TREE = "" ROMS_TREE = "roms" # The emulator folders of a platform whose registry describes them. EMULATORS_TREE = "emulators" +# The frontend's configs folder of a platform whose file records it. +CONFIGS_TREE = "configs" # The tree a platform's list installs into when it starts above the BIOS # folder: a plain dest, with no prefix. INSTALL_TREE = "install" @@ -433,12 +435,18 @@ def handed_directories(config: dict) -> list[HandedDirectory]: source_ref=str(record.get("source_ref", "")), roms_place=None if folder is not None else ( _roms_place(config, folder_in_roms(raw, config)) + or _configs_place(folder_in_configs(raw, config)) or _emulators_place(folder_in_emulators(raw, config)) ), )) return found +def _configs_place(path: str | None) -> Place | None: + """A folder below the configs root, as a place of the configs tree.""" + return None if path is None else (CONFIGS_TREE, path) + + def _emulators_place(path: str | None) -> Place | None: """A folder an emulator key names, as a place of the emulators tree.""" return None if path is None else (EMULATORS_TREE, path) @@ -458,6 +466,18 @@ def _roms_place(config: dict, path: str | None) -> Place | None: return (ROMS_TREE, path) +def folder_in_configs(directory: str, config: dict) -> str | None: + """A directory relative to the frontend's configs folder, None outside it. + + Only a platform whose file records its configs root has one + (``configs_root.path``): Batocera's /userdata/system/configs, where its + generators start shadPS4 from configs/shadps4. + """ + from common import CONFIGS_ROOT # noqa: PLC0415 + + return folder_in_root(directory, config, CONFIGS_ROOT) + + def folder_in_roms(directory: str, config: dict) -> str | None: """A directory relative to the platform's roms directory, None outside it. @@ -465,9 +485,16 @@ def folder_in_roms(directory: str, config: dict) -> str | None: (``roms_root.path``): ROCKNIX's /storage/roms, which holds its BIOS folder and the folders its launchers run programs from. """ - from common import ROMS_ROOT, outer_layout # noqa: PLC0415 + from common import ROMS_ROOT # noqa: PLC0415 - roms = str((outer_layout(config, ROMS_ROOT) or {}).get("path") or "").rstrip("/") + return folder_in_root(directory, config, ROMS_ROOT) + + +def folder_in_root(directory: str, config: dict, root: str) -> str | None: + """A directory relative to an outer root's path, None outside it.""" + from common import outer_layout # noqa: PLC0415 + + roms = str((outer_layout(config, root) or {}).get("path") or "").rstrip("/") directory = directory.strip().rstrip("/") if not roms or not directory.startswith(roms + "/"): return None diff --git a/scripts/scraper/batocera_launchers.py b/scripts/scraper/batocera_launchers.py index aa4222ac9..45f84bd60 100644 --- a/scripts/scraper/batocera_launchers.py +++ b/scripts/scraper/batocera_launchers.py @@ -97,6 +97,32 @@ def path_constants(source: Source) -> dict[str, str]: return constants +def configs_root(cg: Configgen) -> dict[str, str]: + """Where configgen keeps the emulators' configs, relative to the BIOS folder. + + batoceraPaths' CONFIGS (/userdata/system/configs), the folder a generator + starts an emulator from (shadPS4's configs/shadps4), with the lines that + build it and BIOS. + """ + import posixpath # noqa: PLC0415 + + src = cg.source(f"{cg.generators.rsplit('/', 1)[0]}/batoceraPaths.py") + if "CONFIGS" not in cg.paths: + src.fail("no CONFIGS path constant") + wanted: set[str] = {"CONFIGS", "BIOS"} + spans = [] + for node in reversed(src.tree.body): + name = _assigned_name(node) + if name in wanted: + spans.append((node.lineno, node.end_lineno or node.lineno)) + wanted |= {n.id for n in ast.walk(node.value) if isinstance(n, ast.Name)} # type: ignore[union-attr] + return { + "path": cg.paths["CONFIGS"], + "from_bios": posixpath.relpath(cg.paths["CONFIGS"], cg.bios), + "source_ref": src.ref(spans), + } + + def _path_value(node: ast.AST | None, constants: dict[str, str]) -> str | None: if isinstance(node, ast.Name): return constants.get(node.id) diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index 399d8bd43..8ef10db16 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -29,7 +29,7 @@ from .base_scraper import ( md5_field, requirement_entry, ) -from .batocera_launchers import extract_launchers +from .batocera_launchers import Configgen, configs_root, extract_launchers PLATFORM_NAME = "batocera" @@ -656,6 +656,9 @@ class Scraper(BaseScraper): # The systems each libretro core runs for: its save folders. result["libretro_systems"] = libretro_systems result["launchers"] = launchers + result["configs_root"] = configs_root( + Configgen(lambda path: self._fetch_text(f"{_RAW_BASE}/{self.tag}/{path}")) + ) return result diff --git a/scripts/scraper/knulli_scraper.py b/scripts/scraper/knulli_scraper.py index 80068ebd9..763c25ec4 100644 --- a/scripts/scraper/knulli_scraper.py +++ b/scripts/scraper/knulli_scraper.py @@ -49,7 +49,9 @@ from .batocera_scraper import ( submodule_revision, ) from .es_emulators import attach_system_cores +from .batocera_launchers import configs_root from .knulli_launchers import BATOCERA_SUBMODULE, extract_launchers +from .knulli_launchers import configgen as knulli_configgen PLATFORM_NAME = "knulli" @@ -167,6 +169,9 @@ class Scraper(BaseScraper): # Batocera's does: the systems each core runs for are its save folders. result["libretro_systems"] = es_systems_libretro_systems(es_systems, dropped) result["launchers"] = launchers + result["configs_root"] = configs_root( + knulli_configgen(lambda path: self._fetch_text(_RAW.format(ref=self.tag) + path)) + ) return result diff --git a/scripts/verify.py b/scripts/verify.py index 7a05a1be9..beaeb9b6b 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -980,6 +980,7 @@ _UNPLACED_REASONS = { "outside_bios_folder": "read in a folder the platform keeps outside the BIOS tree", "no_roms_directory": "read below a roms folder the platform's registry does not name", "no_emulators_directory": "read in an emulator folder the platform's registry does not describe", + "no_configs_directory": "read below a configs folder the platform's file does not record", "no_platform_slug": "no platform slug for its system", "below_platform_folder": "below the folder the platform lists", "no_save_directory": "read below a save directory the platform keeps in no fixed place", diff --git a/tests/test_install.py b/tests/test_install.py index 9ca17cabb..3f475b166 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1150,7 +1150,7 @@ class OuterRootsAreUntrusted(unittest.TestCase): def test_unsafe_from_bios_is_refused(self): for value in (*self.UNSAFE, "../../../x", "../x/..", "x", "./x", "/x", "..x", None): with self.subTest(from_bios=value), self.assertRaises(ValueError): - install._check_roms_directory({"roms_directory": {**self.ROMS, "from_bios": value}}) + install._check_relative_directories({"roms_directory": {**self.ROMS, "from_bios": value}}) def test_the_relations_the_rom_folders_files_use_are_accepted(self): bios = Path("/card/system/bios") @@ -1161,13 +1161,13 @@ class OuterRootsAreUntrusted(unittest.TestCase): } for value, root in expected.items(): with self.subTest(from_bios=value): - install._check_roms_directory({"roms_directory": {**self.ROMS, "from_bios": value}}) - self.assertEqual(install._roms_root(bios, value), root) + install._check_relative_directories({"roms_directory": {**self.ROMS, "from_bios": value}}) + self.assertEqual(install._relative_root(bios, value), root) self.assertEqual(install.MAX_ROMS_UPS, 2) def test_a_list_without_a_roms_root_leaves_the_files_out(self): files = [{"dest": "pico-8/pico8_64", "root": "roms", "cores": ["x"]}] - kept, left = install.place_roms_entries(files, Path("/card/roms/bios"), {}, "x", say=False) + kept, left = install.place_relative_entries(files, Path("/card/roms/bios"), {}, "x", say=False) self.assertEqual((kept, [o["reason"] for o in left]), ([], ["no_roms_directory"])) def test_unsafe_emulator_folders_are_refused(self): @@ -1192,7 +1192,7 @@ class OuterRootsAreUntrusted(unittest.TestCase): with unittest.mock.patch.object(install.os, "name", host): for value in ("../C:x", "..\\x", "../x:", "../Roms:x"): with self.subTest(host=host, from_bios=value), self.assertRaises(ValueError): - install._check_roms_directory( + install._check_relative_directories( {"roms_directory": {**self.ROMS, "from_bios": value}}) for value in ("C:x/{key}", "x:/{key}", "emu:x/{key}"): layout = {**self.EMULATORS, "searched": [value], "default": value} diff --git a/tests/test_launcher_paths.py b/tests/test_launcher_paths.py index 0b978bb76..261de19b8 100644 --- a/tests/test_launcher_paths.py +++ b/tests/test_launcher_paths.py @@ -253,5 +253,45 @@ class ADefaultTheBuildLacksIsNotRun(unittest.TestCase): self.assertNotIn("suyu", config.get("standalone_cores") or []) + +class AUserFolderInTheConfigsFolder(unittest.TestCase): + """Batocera starts shadPS4 from /userdata/system/configs/shadps4, a folder + of the configs root its batoceraPaths builds (../system/configs).""" + + def test_the_sys_modules_go_below_the_configs_root(self): + from common import load_emulator_profiles, load_platform_config, platform_layouts + from common import standalone_set + + profile = load_emulator_profiles(str(REPO_ROOT / "emulators"))["shadps4"] + for platform in ("batocera", "knulli"): + config = load_platform_config(platform, str(REPO_ROOT / "platforms")) + ((view, _standalone),) = platform_layouts("shadps4", profile, standalone_set(config)) + with self.subTest(platform=platform): + self.assertEqual( + {(f.get("root"), f["standalone_path"].rsplit("/", 1)[0]) for f in view["files"]}, + {("configs", "shadps4/user/sys_modules"), ("configs", "shadps4/user/fonts/font")}, + ) + + def test_the_installer_follows_from_bios_to_the_configs_folder(self): + import install + + manifest = {"configs_directory": {"bios_folder": "bios", "from_bios": "../system/configs"}} + install._check_relative_directories(manifest) + files = [{"dest": "shadps4/user/sys_modules/libSceNgs2.sprx", "root": "configs"}] + bios = Path("/userdata/bios") + kept, left = install.place_relative_entries(files, bios, manifest, "x", say=False) + self.assertEqual((len(kept), left), (1, [])) + self.assertEqual( + install._entry_place(bios, kept[0])[0], Path("/userdata/system/configs") + ) + kept, left = install.place_relative_entries( + files, Path("/elsewhere/firmware"), manifest, "x", say=False) + self.assertEqual((kept, [o["reason"] for o in left]), ([], ["no_configs_directory"])) + for value in ("../C:x", "../../../x", "../system/../x"): + with self.subTest(from_bios=value), self.assertRaises(ValueError): + install._check_relative_directories( + {"configs_directory": {"bios_folder": "bios", "from_bios": value}}) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_launcher_placement.py b/tests/test_launcher_placement.py index 2e23f5c75..cf0e2a55e 100644 --- a/tests/test_launcher_placement.py +++ b/tests/test_launcher_placement.py @@ -383,10 +383,10 @@ class AProgramRunFromItsFolder(unittest.TestCase): import install manifest = self.manifest("romsplat") - install._check_roms_directory(manifest) + install._check_relative_directories(manifest) files = list(manifest["files"]) card = self.root / "card" / "roms" / "bios" - kept, left = install.place_roms_entries(files, card, manifest, "romsplat", say=False) + kept, left = install.place_relative_entries(files, card, manifest, "romsplat", say=False) self.assertEqual((len(kept), left), (2, [])) self.assertEqual( install._entry_path(card, kept[0]), @@ -394,22 +394,22 @@ class AProgramRunFromItsFolder(unittest.TestCase): if install._entry_path(card, kept[0]).is_absolute() else card.parent / kept[0]["dest"], ) other = self.root / "elsewhere" - kept, left = install.place_roms_entries(files, other, manifest, "romsplat", say=False) + kept, left = install.place_relative_entries(files, other, manifest, "romsplat", say=False) self.assertEqual(kept, []) self.assertEqual({o["reason"] for o in left}, {"no_roms_directory"}) with self.assertRaises(ValueError): - install._check_roms_directory({"files": files}) + install._check_relative_directories({"files": files}) # A roms root beside the BIOS folder (Batocera's ../roms). beside = {**manifest, "roms_directory": {"bios_folder": "bios", "from_bios": "../roms"}} - install._check_roms_directory(beside) + install._check_relative_directories(beside) disk = self.root / "userdata" / "bios" - install.place_roms_entries(files, disk, beside, "romsplat", say=False) + install.place_relative_entries(files, disk, beside, "romsplat", say=False) self.assertEqual(install._entry_place(disk, kept_entry := files[0])[0], self.root / "userdata" / "roms") self.assertEqual(kept_entry["root"], "roms") for climbing in ("roms", "../roms/..", "/storage/roms"): with self.subTest(from_bios=climbing), self.assertRaises(ValueError): - install._check_roms_directory( + install._check_relative_directories( {**manifest, "roms_directory": {"bios_folder": "bios", "from_bios": climbing}} )