From 0becfce2b5a1f5fe26a7efd7d558b8ba5c96d66b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:17:01 +0200 Subject: [PATCH] feat: install save-dir files per system folder --- install.ps1 | 2 +- install.py | 57 ++++++--- install.sh | 2 +- tests/test_save_root.py | 273 +++++++++++++++++++++++++++++++++++++++- 4 files changed, 315 insertions(+), 19 deletions(-) diff --git a/install.ps1 b/install.ps1 index 8b8de9a6..fb7d9bda 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "b574c5beb84e9988a53a742f1abf0ad98fcb23dae961facef98c955856380704" + $defaultInstallSha256 = "ad3b338a8f709be45352a12bd9c6a6250819a8fb9afd19498e24f41e3f062d20" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index e4df7bd5..986d2133 100644 --- a/install.py +++ b/install.py @@ -107,6 +107,7 @@ OMISSION_REASONS = { "below_platform_folder": "below the platform folder", "no_save_directory": "no save directory", "no_core_folder": "no core folder", + "no_save_system": "no system runs the core", } # What a file is when it is not firmware, as the manifest's category field # says, and as the installer words it. Firmware carries no category. @@ -2447,6 +2448,15 @@ def _entry_path(root: Path, entry: dict) -> Path: return _destination_path(*_entry_place(root, entry)) +def _entry_key(entry: dict) -> str: + """What tells one entry of the list from another: its dest, below the + BIOS folder or below the save directory. KNULLI lists its own + fbneo/hiscore.dat in the BIOS tree and FB Alpha 2012's below the save + directory, two files under one dest. NUL never appears in a dest.""" + dest = str(entry.get("dest")) + return f"save\0{dest}" if entry.get("root") == "save" else dest + + def _check_root(entry: dict, dest: str) -> None: """root is absent or save, and core_folder one folder name of a save entry.""" root = entry.get("root") @@ -2480,9 +2490,10 @@ def _check_save_directory(data: dict) -> None: return if ( not isinstance(layout, dict) - or set(layout) != {"folder", "by_core"} + or set(layout) - {"by_system"} != {"folder", "by_core"} or not isinstance(layout["by_core"], bool) - or len(_safe_relative_path(layout["folder"], "save_directory.folder").parts) != 1 + or layout.get("by_system", True) is not True + or not _safe_relative_path(layout["folder"], "save_directory.folder").parts ): raise ValueError("invalid save_directory") if layout["by_core"]: @@ -2625,9 +2636,10 @@ def _check_entry( if not isinstance(entry, dict): raise ValueError(f"{where} must be an object") dest = str(_safe_relative_path(entry.get("dest"), f"{where}.dest")) - if dest in seen_destinations: + key = _entry_key({**entry, "dest": dest}) + if key in seen_destinations: raise ValueError(f"duplicate manifest destination: {dest}") - seen_destinations.add(dest) + seen_destinations.add(key) _check_root(entry, dest) size = _check_download(entry, where, dest) @@ -2762,9 +2774,10 @@ def _validate_manifest(data: object, plat: str) -> dict: entry.get("dest"), f"omitted_files[{index}].dest" ) ) - if dest in seen_destinations or dest in seen_omitted: + key = _entry_key({**entry, "dest": dest}) + if key in seen_destinations or key in seen_omitted: raise ValueError(f"duplicate or conflicting omitted destination: {dest}") - seen_omitted.add(dest) + seen_omitted.add(key) _check_omission(entry, dest) _check_root(entry, dest) claims = entry.get("alternatives", []) @@ -3197,7 +3210,7 @@ def region_drops( systems: list[str] | None = None, held: set[str] | None = None, ) -> set[str]: - """Destinations a region priority list withdraws. + """Destinations a region priority list withdraws, as _entry_key spells them. The pack builder's rule (region.resolve_region_drops), replayed over the groups the manifest records. Per group, the best-ranked regional file @@ -3231,7 +3244,7 @@ def region_drops( if not isinstance(priority, int) or isinstance(priority, bool): priority = None for group in joined: - groups.setdefault(group, []).append((set(regions), entry["dest"], priority)) + groups.setdefault(group, []).append((set(regions), _entry_key(entry), priority)) keep: set[str] = set() drop: set[str] = set() for group_id, members in groups.items(): @@ -3304,10 +3317,12 @@ def narrow( # answers the request but takes no installable file's place, unless # the builder holds it (held: true) and only this download lacks it: # the pack drops the fallback it displaces, and so does this. - held = {f["dest"] for f in files} | {o["dest"] for o in omitted if o.get("held")} + held = {_entry_key(f) for f in files} | { + _entry_key(o) for o in omitted if o.get("held") + } dropped = region_drops(files + omitted, regions, systems, held) - files = [f for f in files if f["dest"] not in dropped] - omitted = [o for o in omitted if o["dest"] not in dropped] + files = [f for f in files if _entry_key(f) not in dropped] + omitted = [o for o in omitted if _entry_key(o) not in dropped] if systems: wanted = set(systems) files = [f for f in files if _chosen(f, wanted)] @@ -3488,8 +3503,12 @@ def save_directory_for( default (config.def.h:1099). savefiles_in_content_dir and sort_savefiles_by_content_enable put the directory beside each game: no single place then. Without a config, the BIOS folder standing at its - default name puts the save directory beside it, as the pack lays them out. - ``override`` (--save-dir) names the directory itself. + default name puts the save directory beside it, as the pack lays them out, + and a list whose dests start above the BIOS folder (RetroDECK) puts it + in the folder the run installs to. ``override`` (--save-dir) names the directory itself. A launcher that + writes the save directory at each launch, one folder per system + (by_system: Batocera, RetroBat), is not read from any retroarch.cfg: + the one on disk holds the last launch's system. """ layout = manifest.get("save_directory") if not layout: @@ -3497,7 +3516,7 @@ def save_directory_for( folder = layout["folder"] listed = bool(layout["by_core"]) by_core = listed - found = _RETROARCH_CONFIGS.get(bios_path) + found = None if layout.get("by_system") else _RETROARCH_CONFIGS.get(bios_path) if found is not None: cfg_path, app_dir = found for key in ("savefiles_in_content_dir", "sort_savefiles_by_content_enable"): @@ -3518,6 +3537,10 @@ def save_directory_for( base = manifest.get("base_destination") or "" if base and bios_path.name == base: return SavePlace(bios_path.parent / folder, by_core, listed) + if not base: + # The dests start below the folder holding the BIOS folder + # (RetroDECK's ~/retrodeck holds bios/ and saves/). + return SavePlace(bios_path / folder, by_core, listed) return (f"{bios_path} is not the frontend's default BIOS folder: " "name its save directory with --save-dir") @@ -3537,7 +3560,11 @@ def place_save_entries( place = save_directory_for(bios_path, manifest, override) if isinstance(place, SavePlace): _SAVE_DIRS[bios_path] = place - where = f"{place.directory}{os.sep}" if place.by_core else str(place.directory) + where = str(place.directory) + if (manifest.get("save_directory") or {}).get("by_system"): + where += f"{os.sep}" + if place.by_core: + where += f"{os.sep}" print(f" {_plural(len(saved), 'file')} read from {display}'s save directory " f"go to {where}") return files, [] diff --git a/install.sh b/install.sh index f549bae5..376ff2e1 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="b574c5beb84e9988a53a742f1abf0ad98fcb23dae961facef98c955856380704" +DEFAULT_INSTALL_SHA256="ad3b338a8f709be45352a12bd9c6a6250819a8fb9afd19498e24f41e3f062d20" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_save_root.py b/tests/test_save_root.py index 68489e00..f130f894 100644 --- a/tests/test_save_root.py +++ b/tests/test_save_root.py @@ -96,7 +96,8 @@ class ThePlatformNamesItsSaveDirectory(unittest.TestCase): save_layout({"save_directory": {"folder": "saves", "by_core": True}}), SaveLayout("saves", True), ) - for block in (None, {}, {"folder": ""}, {"folder": "a/b", "by_core": True}): + for block in (None, {}, {"folder": ""}, {"folder": "../b", "by_core": True}, + {"folder": "a/../b", "by_core": True}): with self.subTest(block=block): self.assertIsNone(save_layout({"save_directory": block})) self.assertIsNone(save_layout({})) @@ -126,6 +127,57 @@ class ThePlatformNamesItsSaveDirectory(unittest.TestCase): # RetroPie inherits RetroArch but keeps saves beside each game. self.assertIsNone(save_layout(load_platform_config("retropie", platforms))) + def test_each_platform_running_these_cores_says_where(self): + """The layout read from each platform's launch code, or none where + saves sit beside each game or in a browser.""" + from common import load_platform_config + + platforms = str(REPO_ROOT / "platforms") + expected = { + "batocera": SaveLayout("saves", False, True), + "knulli": SaveLayout("saves", False, True), + "recalbox": SaveLayout("saves", False, True), + "retrodeck": SaveLayout("saves", False, True), + "retrobat": SaveLayout("saves", False, True, frozenset({"dolphin"})), + "emudeck": SaveLayout("saves/retroarch/saves", False), + "rocknix": None, "amberelec": None, "emuelec": None, "romm": None, + "retropie": None, + } + for name, layout in expected.items(): + with self.subTest(platform=name): + config = load_platform_config(name, platforms) + self.assertEqual(save_layout(config), layout) + if layout is not None and layout.by_system: + self.assertTrue(config.get("libretro_systems")) + + def test_a_save_folder_per_system(self): + layout = SaveLayout("saves", False, True) + self.assertEqual(save_destination("User/GBA/gba_bios.bin", "dolphin-emu", layout, + "gamecube"), + "saves/gamecube/User/GBA/gba_bios.bin") + # The system folder is required, and one folder name. + self.assertIsNone(save_destination("a.bin", "X", layout)) + self.assertIsNone(save_destination("a.bin", "X", layout, "a/b")) + # A launcher sorting one core by core (RetroBat's dolphin). + sorted_one = SaveLayout("saves", False, True, frozenset({"dolphin"})) + self.assertTrue(sorted_one.sorts({"dolphin", "dolphin-gui"})) + self.assertFalse(sorted_one.sorts({"stella"})) + self.assertEqual( + save_destination("User/GBA/gba_bios.bin", "dolphin-emu", sorted_one, "wii", + sorted_one.sorts({"dolphin"})), + "saves/wii/dolphin-emu/User/GBA/gba_bios.bin", + ) + + def test_the_systems_a_core_runs_for(self): + from packpaths import libretro_systems + + config = {"libretro_systems": {"dolphin": ["wii", "gamecube"], "fbalpha": ["fbneo"], + "fbalpha2012": ["cps3", "fbneo"]}} + self.assertEqual(libretro_systems(config, ["dolphin"]), ["gamecube", "wii"]) + self.assertEqual(libretro_systems(config, ["fbalpha2012", "fbalpha"]), + ["cps3", "fbneo"]) + self.assertEqual(libretro_systems(config, ["stella"]), []) + class _Fixture(unittest.TestCase): """A RetroArch-like platform with a save directory, one without, and a @@ -300,6 +352,108 @@ class ThePackPlacesThemInTheSaveFolder(_Fixture): self.assertEqual(set(report["pack_destinations"]), set(record.files)) +class AFolderPerSystem(_Fixture): + """Batocera's launcher hands a core /userdata/saves/, RetroBat's + saves\\ with dolphin alone sorted by core: one copy of the file + below each system the platform runs the core for.""" + + OWN = b"COREY OWN FILE" + + def setUp(self) -> None: + super().setUp() + (self.bios / "X" / "own.bin").write_bytes(self.OWN) + sha1 = hashlib.sha1(self.OWN).hexdigest() + self.db["files"][sha1] = { + "path": str(self.bios / "X" / "own.bin"), "name": "own.bin", + "md5": hashlib.md5(self.OWN).hexdigest(), "size": len(self.OWN), "crc32": "", + } + self.db["indexes"]["by_name"]["own.bin"] = [sha1] + (self.emulators / "corey.yml").write_text(yaml.safe_dump({ + "emulator": "CoreY", "type": "libretro", "library_name": "CoreY", + "systems": ["test-console"], + "files": [{"name": "own.bin", "path": "own.bin", "load_from": "save_dir", + "required": False, "sha1": sha1}], + })) + system = {"files": [{"name": "keys.txt", "destination": "keys.txt", + "sha1": hashlib.sha1(self.OTHER_KEYS).hexdigest(), + "required": False}]} + for name in ("testbato", "testrb"): + (self.platforms / f"{name}.yml").write_text(yaml.safe_dump({ + "platform": name, "verification_mode": "existence", + "base_destination": "bios", "cores": ["corex", "corey"], + "systems": {"test-console": system}, + # Two systems run CoreX, none runs CoreY. + "libretro_systems": {"corex": ["sysb", "sysa"], "other": ["sysc"]}, + })) + registry = yaml.safe_load((self.platforms / "_registry.yml").read_text()) + registry["platforms"]["testbato"] = {"save_directory": { + "folder": "saves", "by_core": False, "by_system": True}} + registry["platforms"]["testrb"] = {"save_directory": { + "folder": "saves", "by_core": False, "by_system": True, "by_core_for": ["corex"]}} + (self.platforms / "_registry.yml").write_text(yaml.safe_dump(registry)) + + def test_one_copy_below_each_system(self): + names = self.pack("testbato") + saved = sorted(n for n in names if n.startswith("saves/")) + self.assertEqual(saved, ["saves/sysa/CoreX/sysdata/keys.txt", + "saves/sysb/CoreX/sysdata/keys.txt"]) + self.assertIn("the folder of each system", self.readme) + report = self.verify("testbato") + written = {n for n in names if n != "README.txt" and not n.endswith("/")} + self.assertEqual(set(report["pack_destinations"]), written) + + def test_the_manifest_lists_each_copy(self): + manifest = self.manifest("testbato") + saved = sorted((e["dest"], e.get("core_folder")) for e in manifest["files"] + if e.get("root") == "save") + # The launcher fixes the path: no core folder for the installer to undo. + self.assertEqual(saved, [("sysa/CoreX/sysdata/keys.txt", None), + ("sysb/CoreX/sysdata/keys.txt", None)]) + self.assertEqual(manifest["save_directory"], + {"folder": "saves", "by_core": False, "by_system": True}) + omitted = [(e["dest"], e["reason"]) for e in manifest["omitted_files"] + if e.get("root") == "save"] + self.assertEqual(omitted, [("CoreY/own.bin", "no_save_system")]) + for entry in manifest["files"]: + entry["repo_path"] = f"bios/{Path(entry['repo_path']).name}" + install._validate_manifest(manifest, "testbato") + + def test_verify_says_why_a_core_without_a_system_is_left_out(self): + report = self.verify("testbato") + entry = next(u for u in report["undeclared_files"] if u["name"] == "own.bin") + self.assertFalse(entry["in_pack"]) + self.assertIn("no system runs the core", entry["omitted"]) + + def test_a_core_the_launcher_sorts(self): + names = self.pack("testrb") + self.assertEqual(sorted(n for n in names if n.startswith("saves/")), + ["saves/sysa/CoreX/CoreX/sysdata/keys.txt", + "saves/sysb/CoreX/CoreX/sysdata/keys.txt"]) + manifest = self.manifest("testrb") + self.assertEqual(sorted(e["dest"] for e in manifest["files"] if e.get("root") == "save"), + ["sysa/CoreX/CoreX/sysdata/keys.txt", + "sysb/CoreX/CoreX/sysdata/keys.txt"]) + + def test_two_cores_reading_one_path_in_one_folder_share_it(self): + (self.emulators / "corez.yml").write_text(yaml.safe_dump({ + "emulator": "CoreZ", "type": "libretro", "library_name": "CoreZ", + "systems": ["test-console"], + "files": [{"name": "keys.txt", "path": "CoreX/sysdata/keys.txt", + "load_from": "save_dir", "required": False, + "sha1": hashlib.sha1(self.KEYS).hexdigest()}], + })) + config = yaml.safe_load((self.platforms / "testbato.yml").read_text()) + config["cores"].append("corez") + config["libretro_systems"]["corez"] = ["sysa"] + (self.platforms / "testbato.yml").write_text(yaml.safe_dump(config)) + names = self.pack("testbato") + self.assertEqual(names.count("saves/sysa/CoreX/sysdata/keys.txt"), 1) + manifest = self.manifest("testbato") + shared = next(e for e in manifest["files"] + if e["dest"] == "sysa/CoreX/sysdata/keys.txt") + self.assertEqual(sorted(shared["cores"]), ["corex", "corez"]) + + class TheInstallerFindsTheSaveDirectory(unittest.TestCase): MANIFEST = {"base_destination": "system", "save_directory": {"folder": "saves", "by_core": True}} @@ -349,6 +503,36 @@ class TheInstallerFindsTheSaveDirectory(unittest.TestCase): self.assertEqual(install.save_directory_for(other, self.MANIFEST, self.root / "s"), install.SavePlace(self.root / "s", True)) + def test_a_launcher_writing_it_per_system_is_not_read_from_a_config(self): + """Batocera and RetroBat write retroarch.cfg at each launch: the one + on disk names the last system played.""" + manifest = {"base_destination": "system", + "save_directory": {"folder": "saves", "by_core": False, "by_system": True}} + other = self.root / "elsewhere" + other.mkdir() + self.cfg.write_text(f'savefile_directory = "{other}"\nsort_savefiles_enable = "true"\n') + install._RETROARCH_CONFIGS[self.bios] = (self.cfg, self.app) + self.assertEqual(install.save_directory_for(self.bios, manifest), + install.SavePlace(self.app / "saves", False, False)) + + def test_a_list_rooted_above_the_bios_folder(self): + """RetroDECK's dests start at ~/retrodeck, which holds bios/ and saves/.""" + manifest = {"base_destination": "", + "save_directory": {"folder": "saves", "by_core": False, "by_system": True}} + root = self.root / "retrodeck" + self.assertEqual(install.save_directory_for(root, manifest), + install.SavePlace(root / "saves", False, False)) + + def test_a_save_folder_several_levels_down(self): + """EmuDeck: ~/Emulation/bios and ~/Emulation/saves/retroarch/saves.""" + manifest = {"base_destination": "bios", + "save_directory": {"folder": "saves/retroarch/saves", "by_core": False}} + bios = self.root / "Emulation" / "bios" + self.assertEqual(install.save_directory_for(bios, manifest), + install.SavePlace(self.root / "Emulation" / "saves/retroarch/saves", + False, False)) + install._check_save_directory(manifest) + def test_the_entry_path(self): saves = (self.app / "saves").resolve() install._SAVE_DIRS[self.bios] = install.SavePlace(self.app / "saves", True) @@ -403,7 +587,12 @@ class TheManifestBoundary(unittest.TestCase): ("core folder on a BIOS file", self.manifest( {k: v for k, v in self.ENTRY.items() if k != "root"}, layout)), ("climbing save folder", self.manifest(self.ENTRY, {"folder": "..", "by_core": True})), - ("save folder with a path", self.manifest(self.ENTRY, {"folder": "a/b", "by_core": True})), + ("save folder climbing inside", self.manifest( + self.ENTRY, {"folder": "a/../b", "by_core": True})), + ("by_system not true", self.manifest( + self.ENTRY, {"folder": "saves", "by_core": True, "by_system": False})), + ("unknown save_directory key", self.manifest( + self.ENTRY, {"folder": "saves", "by_core": True, "by_game": True})), ("by_core not a flag", self.manifest(self.ENTRY, {"folder": "saves", "by_core": 1})), ("dest without its core folder", self.manifest( {**self.ENTRY, "dest": "sysdata/keys.txt"}, layout)), @@ -528,6 +717,86 @@ class TheProfilesFollowTheContract(unittest.TestCase): self.assertTrue(all(e["root"] == "save" for e in azahar.values())) +class TheScrapersReadTheSystemLists(unittest.TestCase): + """libretro_systems, read from each platform's own system list.""" + + def setUp(self) -> None: + sys.path.insert(0, str(REPO_ROOT)) + self.addCleanup(sys.path.remove, str(REPO_ROOT)) + + def test_batocera_and_knulli_es_systems_yml(self): + from scripts.scraper.batocera_scraper import es_systems_libretro_systems + + text = ( + "gamecube:\n emulators:\n dolphin:\n dolphin: { requireAnyOf: [A] }\n" + " libretro:\n dolphin: { requireAnyOf: [B] }\n" + "wii:\n emulators:\n libretro:\n dolphin: { requireAnyOf: [B] }\n" + " plugin: { requireAnyOf: [C] }\n" + "atari2600:\n emulators:\n libretro:\n stella: { requireAnyOf: [D] }\n" + ) + self.assertEqual(es_systems_libretro_systems(text, {"plugin"}), + {"dolphin": ["gamecube", "wii"], "stella": ["atari2600"]}) + + def test_retrobat_es_systems_cfg(self): + from scripts.scraper.retrobat_scraper import es_systems_libretro_systems + + text = ( + "3ds" + "azahar" + "" + "citra-qt" + "" + ) + self.assertEqual(es_systems_libretro_systems(text), {"azahar": ["3ds"]}) + + def test_recalbox_system_ini(self): + from scripts.scraper.recalbox_scraper import libretro_systems + + wii = ("[system]\nname = \"wii\"\n[core.0]\nemulator = \"dolphin-gui\"\n" + "core = \"dolphin-gui\"\n[core.2]\nemulator = \"libretro\"\ncore = \"dolphin\"\n") + nameless = "[core.0]\nemulator = \"libretro\"\ncore = \"stella\"\n" + self.assertEqual(libretro_systems({"wii": wii, "atari2600": nameless}), + {"dolphin": ["wii"], "stella": ["atari2600"]}) + + def test_retrodeck_es_de_commands(self): + from scripts.scraper.retrodeck_scraper import es_systems_libretro_systems + + text = ( + "n3ds%ROMPATH%/n3ds" + "%EMULATOR_AZAHAR% %ROM%" + "%EMULATOR_RETROARCH% -L " + "%CORE_RETROARCH%/citra_libretro.so %ROM%" + "" + "" + ) + self.assertEqual(es_systems_libretro_systems(text), {"citra": ["n3ds"]}) + + def test_the_published_platform_files(self): + expected = { + "batocera": {"dolphin": ["gamecube", "wii"], "stella": ["atari2600"]}, + "knulli": {"dolphin": ["gamecube", "wii"], "citra": ["3ds"]}, + "retrobat": {"azahar": ["3ds"], "dolphin": ["gamecube", "triforce", "wii"]}, + "recalbox": {"dolphin": ["gamecube", "wii"], "stella": ["atari2600"]}, + "retrodeck": {"citra": ["n3ds"], "stella": ["atari2600"]}, + } + for name, cores in expected.items(): + table = yaml.safe_load( + (REPO_ROOT / "platforms" / f"{name}.yml").read_text(encoding="utf-8") + )["libretro_systems"] + for core, systems in cores.items(): + with self.subTest(platform=name, core=core): + self.assertEqual(table[core], systems) + + def test_the_published_batocera_list_places_dolphin_per_system(self): + manifest = json.loads((REPO_ROOT / "install" / "batocera.json").read_text()) + self.assertEqual(manifest.get("save_directory"), + {"folder": "saves", "by_core": False, "by_system": True}) + dests = {e["dest"] for e in manifest["files"] if e.get("root") == "save"} + for system in ("gamecube", "wii"): + self.assertIn(f"{system}/User/GBA/gba_bios.bin", dests) + self.assertIn("atari2600/stella/KVS1.WAV", dests) + + class TheSchemasCarryTheFields(unittest.TestCase): def test_install_manifest_and_profile_schemas(self): try: