feat: install save-dir files per system folder

This commit is contained in:
Abdessamad Derraz committed 2026-10-22 21:17:01 +02:00
1 parent 2d26385efe
commit 0becfce2b5
4 files changed
+315 -19

No files matched your search

+1 -1
View File
@@ -11,7 +11,7 @@
$ErrorActionPreference = "Stop"
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
$defaultInstallSha256 = "b574c5beb84e9988a53a742f1abf0ad98fcb23dae961facef98c955856380704"
$defaultInstallSha256 = "ad3b338a8f709be45352a12bd9c6a6250819a8fb9afd19498e24f41e3f062d20"
$maximumInstallerBytes = 2MB
$installerExit = 0
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
+42 -15
View File
@@ -107,6 +107,7 @@ OMISSION_REASONS = {
"below_platform_folder": "below the platform folder",
"no_save_directory": "no save directory",
"no_core_folder": "no core folder",
"no_save_system": "no system runs the core",
}
# What a file is when it is not firmware, as the manifest's category field
# says, and as the installer words it. Firmware carries no category.
@@ -2447,6 +2448,15 @@ def _entry_path(root: Path, entry: dict) -> Path:
return _destination_path(*_entry_place(root, entry))
def _entry_key(entry: dict) -> str:
"""What tells one entry of the list from another: its dest, below the
BIOS folder or below the save directory. KNULLI lists its own
fbneo/hiscore.dat in the BIOS tree and FB Alpha 2012's below the save
directory, two files under one dest. NUL never appears in a dest."""
dest = str(entry.get("dest"))
return f"save\0{dest}" if entry.get("root") == "save" else dest
def _check_root(entry: dict, dest: str) -> None:
"""root is absent or save, and core_folder one folder name of a save entry."""
root = entry.get("root")
@@ -2480,9 +2490,10 @@ def _check_save_directory(data: dict) -> None:
return
if (
not isinstance(layout, dict)
or set(layout) != {"folder", "by_core"}
or set(layout) - {"by_system"} != {"folder", "by_core"}
or not isinstance(layout["by_core"], bool)
or len(_safe_relative_path(layout["folder"], "save_directory.folder").parts) != 1
or layout.get("by_system", True) is not True
or not _safe_relative_path(layout["folder"], "save_directory.folder").parts
):
raise ValueError("invalid save_directory")
if layout["by_core"]:
@@ -2625,9 +2636,10 @@ def _check_entry(
if not isinstance(entry, dict):
raise ValueError(f"{where} must be an object")
dest = str(_safe_relative_path(entry.get("dest"), f"{where}.dest"))
if dest in seen_destinations:
key = _entry_key({**entry, "dest": dest})
if key in seen_destinations:
raise ValueError(f"duplicate manifest destination: {dest}")
seen_destinations.add(dest)
seen_destinations.add(key)
_check_root(entry, dest)
size = _check_download(entry, where, dest)
@@ -2762,9 +2774,10 @@ def _validate_manifest(data: object, plat: str) -> dict:
entry.get("dest"), f"omitted_files[{index}].dest"
)
)
if dest in seen_destinations or dest in seen_omitted:
key = _entry_key({**entry, "dest": dest})
if key in seen_destinations or key in seen_omitted:
raise ValueError(f"duplicate or conflicting omitted destination: {dest}")
seen_omitted.add(dest)
seen_omitted.add(key)
_check_omission(entry, dest)
_check_root(entry, dest)
claims = entry.get("alternatives", [])
@@ -3197,7 +3210,7 @@ def region_drops(
systems: list[str] | None = None,
held: set[str] | None = None,
) -> set[str]:
"""Destinations a region priority list withdraws.
"""Destinations a region priority list withdraws, as _entry_key spells them.
The pack builder's rule (region.resolve_region_drops), replayed over the
groups the manifest records. Per group, the best-ranked regional file
@@ -3231,7 +3244,7 @@ def region_drops(
if not isinstance(priority, int) or isinstance(priority, bool):
priority = None
for group in joined:
groups.setdefault(group, []).append((set(regions), entry["dest"], priority))
groups.setdefault(group, []).append((set(regions), _entry_key(entry), priority))
keep: set[str] = set()
drop: set[str] = set()
for group_id, members in groups.items():
@@ -3304,10 +3317,12 @@ def narrow(
# answers the request but takes no installable file's place, unless
# the builder holds it (held: true) and only this download lacks it:
# the pack drops the fallback it displaces, and so does this.
held = {f["dest"] for f in files} | {o["dest"] for o in omitted if o.get("held")}
held = {_entry_key(f) for f in files} | {
_entry_key(o) for o in omitted if o.get("held")
}
dropped = region_drops(files + omitted, regions, systems, held)
files = [f for f in files if f["dest"] not in dropped]
omitted = [o for o in omitted if o["dest"] not in dropped]
files = [f for f in files if _entry_key(f) not in dropped]
omitted = [o for o in omitted if _entry_key(o) not in dropped]
if systems:
wanted = set(systems)
files = [f for f in files if _chosen(f, wanted)]
@@ -3488,8 +3503,12 @@ def save_directory_for(
default (config.def.h:1099). savefiles_in_content_dir and
sort_savefiles_by_content_enable put the directory beside each game: no
single place then. Without a config, the BIOS folder standing at its
default name puts the save directory beside it, as the pack lays them out.
``override`` (--save-dir) names the directory itself.
default name puts the save directory beside it, as the pack lays them out,
and a list whose dests start above the BIOS folder (RetroDECK) puts it
in the folder the run installs to. ``override`` (--save-dir) names the directory itself. A launcher that
writes the save directory at each launch, one folder per system
(by_system: Batocera, RetroBat), is not read from any retroarch.cfg:
the one on disk holds the last launch's system.
"""
layout = manifest.get("save_directory")
if not layout:
@@ -3497,7 +3516,7 @@ def save_directory_for(
folder = layout["folder"]
listed = bool(layout["by_core"])
by_core = listed
found = _RETROARCH_CONFIGS.get(bios_path)
found = None if layout.get("by_system") else _RETROARCH_CONFIGS.get(bios_path)
if found is not None:
cfg_path, app_dir = found
for key in ("savefiles_in_content_dir", "sort_savefiles_by_content_enable"):
@@ -3518,6 +3537,10 @@ def save_directory_for(
base = manifest.get("base_destination") or ""
if base and bios_path.name == base:
return SavePlace(bios_path.parent / folder, by_core, listed)
if not base:
# The dests start below the folder holding the BIOS folder
# (RetroDECK's ~/retrodeck holds bios/ and saves/).
return SavePlace(bios_path / folder, by_core, listed)
return (f"{bios_path} is not the frontend's default BIOS folder: "
"name its save directory with --save-dir")
@@ -3537,7 +3560,11 @@ def place_save_entries(
place = save_directory_for(bios_path, manifest, override)
if isinstance(place, SavePlace):
_SAVE_DIRS[bios_path] = place
where = f"{place.directory}{os.sep}<core>" if place.by_core else str(place.directory)
where = str(place.directory)
if (manifest.get("save_directory") or {}).get("by_system"):
where += f"{os.sep}<system>"
if place.by_core:
where += f"{os.sep}<core>"
print(f" {_plural(len(saved), 'file')} read from {display}'s save directory "
f"go to {where}")
return files, []
+1 -1
View File
@@ -18,7 +18,7 @@ esac
TEMP_INSTALLER=""
TEMP_DIRECTORY=""
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
DEFAULT_INSTALL_SHA256="b574c5beb84e9988a53a742f1abf0ad98fcb23dae961facef98c955856380704"
DEFAULT_INSTALL_SHA256="ad3b338a8f709be45352a12bd9c6a6250819a8fb9afd19498e24f41e3f062d20"
MAX_INSTALLER_BYTES=2097152
cleanup() {
+271 -2
View File
@@ -96,7 +96,8 @@ class ThePlatformNamesItsSaveDirectory(unittest.TestCase):
save_layout({"save_directory": {"folder": "saves", "by_core": True}}),
SaveLayout("saves", True),
)
for block in (None, {}, {"folder": ""}, {"folder": "a/b", "by_core": True}):
for block in (None, {}, {"folder": ""}, {"folder": "../b", "by_core": True},
{"folder": "a/../b", "by_core": True}):
with self.subTest(block=block):
self.assertIsNone(save_layout({"save_directory": block}))
self.assertIsNone(save_layout({}))
@@ -126,6 +127,57 @@ class ThePlatformNamesItsSaveDirectory(unittest.TestCase):
# RetroPie inherits RetroArch but keeps saves beside each game.
self.assertIsNone(save_layout(load_platform_config("retropie", platforms)))
def test_each_platform_running_these_cores_says_where(self):
"""The layout read from each platform's launch code, or none where
saves sit beside each game or in a browser."""
from common import load_platform_config
platforms = str(REPO_ROOT / "platforms")
expected = {
"batocera": SaveLayout("saves", False, True),
"knulli": SaveLayout("saves", False, True),
"recalbox": SaveLayout("saves", False, True),
"retrodeck": SaveLayout("saves", False, True),
"retrobat": SaveLayout("saves", False, True, frozenset({"dolphin"})),
"emudeck": SaveLayout("saves/retroarch/saves", False),
"rocknix": None, "amberelec": None, "emuelec": None, "romm": None,
"retropie": None,
}
for name, layout in expected.items():
with self.subTest(platform=name):
config = load_platform_config(name, platforms)
self.assertEqual(save_layout(config), layout)
if layout is not None and layout.by_system:
self.assertTrue(config.get("libretro_systems"))
def test_a_save_folder_per_system(self):
layout = SaveLayout("saves", False, True)
self.assertEqual(save_destination("User/GBA/gba_bios.bin", "dolphin-emu", layout,
"gamecube"),
"saves/gamecube/User/GBA/gba_bios.bin")
# The system folder is required, and one folder name.
self.assertIsNone(save_destination("a.bin", "X", layout))
self.assertIsNone(save_destination("a.bin", "X", layout, "a/b"))
# A launcher sorting one core by core (RetroBat's dolphin).
sorted_one = SaveLayout("saves", False, True, frozenset({"dolphin"}))
self.assertTrue(sorted_one.sorts({"dolphin", "dolphin-gui"}))
self.assertFalse(sorted_one.sorts({"stella"}))
self.assertEqual(
save_destination("User/GBA/gba_bios.bin", "dolphin-emu", sorted_one, "wii",
sorted_one.sorts({"dolphin"})),
"saves/wii/dolphin-emu/User/GBA/gba_bios.bin",
)
def test_the_systems_a_core_runs_for(self):
from packpaths import libretro_systems
config = {"libretro_systems": {"dolphin": ["wii", "gamecube"], "fbalpha": ["fbneo"],
"fbalpha2012": ["cps3", "fbneo"]}}
self.assertEqual(libretro_systems(config, ["dolphin"]), ["gamecube", "wii"])
self.assertEqual(libretro_systems(config, ["fbalpha2012", "fbalpha"]),
["cps3", "fbneo"])
self.assertEqual(libretro_systems(config, ["stella"]), [])
class _Fixture(unittest.TestCase):
"""A RetroArch-like platform with a save directory, one without, and a
@@ -300,6 +352,108 @@ class ThePackPlacesThemInTheSaveFolder(_Fixture):
self.assertEqual(set(report["pack_destinations"]), set(record.files))
class AFolderPerSystem(_Fixture):
"""Batocera's launcher hands a core /userdata/saves/<system>, RetroBat's
saves\\<system> with dolphin alone sorted by core: one copy of the file
below each system the platform runs the core for."""
OWN = b"COREY OWN FILE"
def setUp(self) -> None:
super().setUp()
(self.bios / "X" / "own.bin").write_bytes(self.OWN)
sha1 = hashlib.sha1(self.OWN).hexdigest()
self.db["files"][sha1] = {
"path": str(self.bios / "X" / "own.bin"), "name": "own.bin",
"md5": hashlib.md5(self.OWN).hexdigest(), "size": len(self.OWN), "crc32": "",
}
self.db["indexes"]["by_name"]["own.bin"] = [sha1]
(self.emulators / "corey.yml").write_text(yaml.safe_dump({
"emulator": "CoreY", "type": "libretro", "library_name": "CoreY",
"systems": ["test-console"],
"files": [{"name": "own.bin", "path": "own.bin", "load_from": "save_dir",
"required": False, "sha1": sha1}],
}))
system = {"files": [{"name": "keys.txt", "destination": "keys.txt",
"sha1": hashlib.sha1(self.OTHER_KEYS).hexdigest(),
"required": False}]}
for name in ("testbato", "testrb"):
(self.platforms / f"{name}.yml").write_text(yaml.safe_dump({
"platform": name, "verification_mode": "existence",
"base_destination": "bios", "cores": ["corex", "corey"],
"systems": {"test-console": system},
# Two systems run CoreX, none runs CoreY.
"libretro_systems": {"corex": ["sysb", "sysa"], "other": ["sysc"]},
}))
registry = yaml.safe_load((self.platforms / "_registry.yml").read_text())
registry["platforms"]["testbato"] = {"save_directory": {
"folder": "saves", "by_core": False, "by_system": True}}
registry["platforms"]["testrb"] = {"save_directory": {
"folder": "saves", "by_core": False, "by_system": True, "by_core_for": ["corex"]}}
(self.platforms / "_registry.yml").write_text(yaml.safe_dump(registry))
def test_one_copy_below_each_system(self):
names = self.pack("testbato")
saved = sorted(n for n in names if n.startswith("saves/"))
self.assertEqual(saved, ["saves/sysa/CoreX/sysdata/keys.txt",
"saves/sysb/CoreX/sysdata/keys.txt"])
self.assertIn("the folder of each system", self.readme)
report = self.verify("testbato")
written = {n for n in names if n != "README.txt" and not n.endswith("/")}
self.assertEqual(set(report["pack_destinations"]), written)
def test_the_manifest_lists_each_copy(self):
manifest = self.manifest("testbato")
saved = sorted((e["dest"], e.get("core_folder")) for e in manifest["files"]
if e.get("root") == "save")
# The launcher fixes the path: no core folder for the installer to undo.
self.assertEqual(saved, [("sysa/CoreX/sysdata/keys.txt", None),
("sysb/CoreX/sysdata/keys.txt", None)])
self.assertEqual(manifest["save_directory"],
{"folder": "saves", "by_core": False, "by_system": True})
omitted = [(e["dest"], e["reason"]) for e in manifest["omitted_files"]
if e.get("root") == "save"]
self.assertEqual(omitted, [("CoreY/own.bin", "no_save_system")])
for entry in manifest["files"]:
entry["repo_path"] = f"bios/{Path(entry['repo_path']).name}"
install._validate_manifest(manifest, "testbato")
def test_verify_says_why_a_core_without_a_system_is_left_out(self):
report = self.verify("testbato")
entry = next(u for u in report["undeclared_files"] if u["name"] == "own.bin")
self.assertFalse(entry["in_pack"])
self.assertIn("no system runs the core", entry["omitted"])
def test_a_core_the_launcher_sorts(self):
names = self.pack("testrb")
self.assertEqual(sorted(n for n in names if n.startswith("saves/")),
["saves/sysa/CoreX/CoreX/sysdata/keys.txt",
"saves/sysb/CoreX/CoreX/sysdata/keys.txt"])
manifest = self.manifest("testrb")
self.assertEqual(sorted(e["dest"] for e in manifest["files"] if e.get("root") == "save"),
["sysa/CoreX/CoreX/sysdata/keys.txt",
"sysb/CoreX/CoreX/sysdata/keys.txt"])
def test_two_cores_reading_one_path_in_one_folder_share_it(self):
(self.emulators / "corez.yml").write_text(yaml.safe_dump({
"emulator": "CoreZ", "type": "libretro", "library_name": "CoreZ",
"systems": ["test-console"],
"files": [{"name": "keys.txt", "path": "CoreX/sysdata/keys.txt",
"load_from": "save_dir", "required": False,
"sha1": hashlib.sha1(self.KEYS).hexdigest()}],
}))
config = yaml.safe_load((self.platforms / "testbato.yml").read_text())
config["cores"].append("corez")
config["libretro_systems"]["corez"] = ["sysa"]
(self.platforms / "testbato.yml").write_text(yaml.safe_dump(config))
names = self.pack("testbato")
self.assertEqual(names.count("saves/sysa/CoreX/sysdata/keys.txt"), 1)
manifest = self.manifest("testbato")
shared = next(e for e in manifest["files"]
if e["dest"] == "sysa/CoreX/sysdata/keys.txt")
self.assertEqual(sorted(shared["cores"]), ["corex", "corez"])
class TheInstallerFindsTheSaveDirectory(unittest.TestCase):
MANIFEST = {"base_destination": "system",
"save_directory": {"folder": "saves", "by_core": True}}
@@ -349,6 +503,36 @@ class TheInstallerFindsTheSaveDirectory(unittest.TestCase):
self.assertEqual(install.save_directory_for(other, self.MANIFEST, self.root / "s"),
install.SavePlace(self.root / "s", True))
def test_a_launcher_writing_it_per_system_is_not_read_from_a_config(self):
"""Batocera and RetroBat write retroarch.cfg at each launch: the one
on disk names the last system played."""
manifest = {"base_destination": "system",
"save_directory": {"folder": "saves", "by_core": False, "by_system": True}}
other = self.root / "elsewhere"
other.mkdir()
self.cfg.write_text(f'savefile_directory = "{other}"\nsort_savefiles_enable = "true"\n')
install._RETROARCH_CONFIGS[self.bios] = (self.cfg, self.app)
self.assertEqual(install.save_directory_for(self.bios, manifest),
install.SavePlace(self.app / "saves", False, False))
def test_a_list_rooted_above_the_bios_folder(self):
"""RetroDECK's dests start at ~/retrodeck, which holds bios/ and saves/."""
manifest = {"base_destination": "",
"save_directory": {"folder": "saves", "by_core": False, "by_system": True}}
root = self.root / "retrodeck"
self.assertEqual(install.save_directory_for(root, manifest),
install.SavePlace(root / "saves", False, False))
def test_a_save_folder_several_levels_down(self):
"""EmuDeck: ~/Emulation/bios and ~/Emulation/saves/retroarch/saves."""
manifest = {"base_destination": "bios",
"save_directory": {"folder": "saves/retroarch/saves", "by_core": False}}
bios = self.root / "Emulation" / "bios"
self.assertEqual(install.save_directory_for(bios, manifest),
install.SavePlace(self.root / "Emulation" / "saves/retroarch/saves",
False, False))
install._check_save_directory(manifest)
def test_the_entry_path(self):
saves = (self.app / "saves").resolve()
install._SAVE_DIRS[self.bios] = install.SavePlace(self.app / "saves", True)
@@ -403,7 +587,12 @@ class TheManifestBoundary(unittest.TestCase):
("core folder on a BIOS file", self.manifest(
{k: v for k, v in self.ENTRY.items() if k != "root"}, layout)),
("climbing save folder", self.manifest(self.ENTRY, {"folder": "..", "by_core": True})),
("save folder with a path", self.manifest(self.ENTRY, {"folder": "a/b", "by_core": True})),
("save folder climbing inside", self.manifest(
self.ENTRY, {"folder": "a/../b", "by_core": True})),
("by_system not true", self.manifest(
self.ENTRY, {"folder": "saves", "by_core": True, "by_system": False})),
("unknown save_directory key", self.manifest(
self.ENTRY, {"folder": "saves", "by_core": True, "by_game": True})),
("by_core not a flag", self.manifest(self.ENTRY, {"folder": "saves", "by_core": 1})),
("dest without its core folder", self.manifest(
{**self.ENTRY, "dest": "sysdata/keys.txt"}, layout)),
@@ -528,6 +717,86 @@ class TheProfilesFollowTheContract(unittest.TestCase):
self.assertTrue(all(e["root"] == "save" for e in azahar.values()))
class TheScrapersReadTheSystemLists(unittest.TestCase):
"""libretro_systems, read from each platform's own system list."""
def setUp(self) -> None:
sys.path.insert(0, str(REPO_ROOT))
self.addCleanup(sys.path.remove, str(REPO_ROOT))
def test_batocera_and_knulli_es_systems_yml(self):
from scripts.scraper.batocera_scraper import es_systems_libretro_systems
text = (
"gamecube:\n emulators:\n dolphin:\n dolphin: { requireAnyOf: [A] }\n"
" libretro:\n dolphin: { requireAnyOf: [B] }\n"
"wii:\n emulators:\n libretro:\n dolphin: { requireAnyOf: [B] }\n"
" plugin: { requireAnyOf: [C] }\n"
"atari2600:\n emulators:\n libretro:\n stella: { requireAnyOf: [D] }\n"
)
self.assertEqual(es_systems_libretro_systems(text, {"plugin"}),
{"dolphin": ["gamecube", "wii"], "stella": ["atari2600"]})
def test_retrobat_es_systems_cfg(self):
from scripts.scraper.retrobat_scraper import es_systems_libretro_systems
text = (
"<systemList><system><name>3ds</name><emulators>"
"<emulator name=\"libretro\"><cores><core>azahar</core>"
"<!-- <core>citra</core> --></cores></emulator>"
"<emulator name=\"citra-qt\"><cores><core>citra-qt</core></cores></emulator>"
"</emulators></system></systemList>"
)
self.assertEqual(es_systems_libretro_systems(text), {"azahar": ["3ds"]})
def test_recalbox_system_ini(self):
from scripts.scraper.recalbox_scraper import libretro_systems
wii = ("[system]\nname = \"wii\"\n[core.0]\nemulator = \"dolphin-gui\"\n"
"core = \"dolphin-gui\"\n[core.2]\nemulator = \"libretro\"\ncore = \"dolphin\"\n")
nameless = "[core.0]\nemulator = \"libretro\"\ncore = \"stella\"\n"
self.assertEqual(libretro_systems({"wii": wii, "atari2600": nameless}),
{"dolphin": ["wii"], "stella": ["atari2600"]})
def test_retrodeck_es_de_commands(self):
from scripts.scraper.retrodeck_scraper import es_systems_libretro_systems
text = (
"<systemList><system><name>n3ds</name><path>%ROMPATH%/n3ds</path>"
"<command label=\"Azahar (Standalone)\">%EMULATOR_AZAHAR% %ROM%</command>"
"<command label=\"Citra\">%EMULATOR_RETROARCH% -L "
"%CORE_RETROARCH%/citra_libretro.so %ROM%</command>"
"<!-- <command>%EMULATOR_RETROARCH% -L %CORE_RETROARCH%/x_libretro.so</command> -->"
"</system></systemList>"
)
self.assertEqual(es_systems_libretro_systems(text), {"citra": ["n3ds"]})
def test_the_published_platform_files(self):
expected = {
"batocera": {"dolphin": ["gamecube", "wii"], "stella": ["atari2600"]},
"knulli": {"dolphin": ["gamecube", "wii"], "citra": ["3ds"]},
"retrobat": {"azahar": ["3ds"], "dolphin": ["gamecube", "triforce", "wii"]},
"recalbox": {"dolphin": ["gamecube", "wii"], "stella": ["atari2600"]},
"retrodeck": {"citra": ["n3ds"], "stella": ["atari2600"]},
}
for name, cores in expected.items():
table = yaml.safe_load(
(REPO_ROOT / "platforms" / f"{name}.yml").read_text(encoding="utf-8")
)["libretro_systems"]
for core, systems in cores.items():
with self.subTest(platform=name, core=core):
self.assertEqual(table[core], systems)
def test_the_published_batocera_list_places_dolphin_per_system(self):
manifest = json.loads((REPO_ROOT / "install" / "batocera.json").read_text())
self.assertEqual(manifest.get("save_directory"),
{"folder": "saves", "by_core": False, "by_system": True})
dests = {e["dest"] for e in manifest["files"] if e.get("root") == "save"}
for system in ("gamecube", "wii"):
self.assertIn(f"{system}/User/GBA/gba_bios.bin", dests)
self.assertIn("atari2600/stella/KVS1.WAV", dests)
class TheSchemasCarryTheFields(unittest.TestCase):
def test_install_manifest_and_profile_schemas(self):
try: