feat: install save-dir files below the save dir

The installer reads savefile_directory and sort_savefiles_enable from
the retroarch.cfg it found, falls back to saves/ beside a default
system/ folder, and takes --save-dir. Entries it cannot place are said
and left out. The installer hash is re-pinned.
This commit is contained in:
Abdessamad Derraz committed 2026-10-22 19:58:29 +02:00
1 parent 1cd6c9ee4d
commit 7e32db1013
4 files changed
+770 -15

No files matched your search

+1 -1
View File
@@ -11,7 +11,7 @@
$ErrorActionPreference = "Stop"
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
$defaultInstallSha256 = "abc5b672e1d4b2f507b6d8c3d5c23a120a7dffea6e583559bdcc92c9cbbc7692"
$defaultInstallSha256 = "b574c5beb84e9988a53a742f1abf0ad98fcb23dae961facef98c955856380704"
$maximumInstallerBytes = 2MB
$installerExit = 0
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
+215 -13
View File
@@ -105,6 +105,8 @@ OMISSION_REASONS = {
"user_provided": "user provided",
"no_platform_slug": "no platform folder",
"below_platform_folder": "below the platform folder",
"no_save_directory": "no save directory",
"no_core_folder": "no core folder",
}
# What a file is when it is not firmware, as the manifest's category field
# says, and as the installer words it. Firmware carries no category.
@@ -2398,6 +2400,101 @@ def _destination_path(root: Path, value: object) -> Path:
return root.resolve() / Path(*relative.parts)
class SavePlace(NamedTuple):
"""The save directory a frontend's config names, whether it hands each
core a folder of its own below it (RetroArch's sort_savefiles_enable,
runloop.c runloop_path_set_redirect), and whether the file list's dests
hold that folder (its save_directory.by_core)."""
directory: Path
by_core: bool
listed_by_core: bool = True
# BIOS folder -> where the files read below its frontend's save directory go,
# set once the save directory is known (save_directory_for).
_SAVE_DIRS: dict[Path, SavePlace] = {}
def _entry_place(root: Path, entry: dict) -> tuple[Path, object]:
"""The folder an entry's dest is relative to, and the dest.
An entry with root: save is relative to the save directory the frontend
hands the core, below the BIOS folder's frontend save directory and the
core's own folder where the frontend sorts saves by core. Any other is
relative to the BIOS folder.
"""
if entry.get("root") != "save":
return root, entry.get("dest")
place = _SAVE_DIRS.get(root)
if place is None:
raise ValueError(f"no save directory known for {entry.get('dest')}")
dest = entry.get("dest")
folder = entry.get("core_folder")
# No core_folder: the core reads the same place whether the frontend
# sorts saves by core or not (an empty library_name, or Cemu adding its
# own folder only where the directory is not named after it).
if folder and isinstance(dest, str) and place.by_core != place.listed_by_core:
if place.listed_by_core:
dest = dest.split("/", 1)[1] if "/" in dest else dest
else:
dest = f"{folder}/{dest}"
return place.directory, dest
def _entry_path(root: Path, entry: dict) -> Path:
"""Where an entry is written, as _entry_place places it."""
return _destination_path(*_entry_place(root, entry))
def _check_root(entry: dict, dest: str) -> None:
"""root is absent or save, and core_folder one folder name of a save entry."""
root = entry.get("root")
if root is not None and root != "save":
raise ValueError(f"invalid root for {dest}")
folder = entry.get("core_folder")
if folder is None:
return
if root != "save":
raise ValueError(f"core_folder outside root: save for {dest}")
if len(_safe_relative_path(folder, f"core_folder of {dest}").parts) != 1:
raise ValueError(f"core_folder must be one folder name for {dest}")
def _check_save_directory(data: dict) -> None:
"""The manifest's save_directory, which every root: save entry needs."""
layout = data.get("save_directory")
entries = [
entry for field in ("files", "case_variants", "omitted_files")
for entry in data.get(field) or [] if isinstance(entry, dict)
]
if layout is None:
# An omission may name one: the list says the file exists and why
# this platform cannot place it.
installable = [
entry for field in ("files", "case_variants")
for entry in data.get(field) or [] if isinstance(entry, dict)
]
if any(entry.get("root") == "save" for entry in installable):
raise ValueError("entries below the save directory and no save_directory")
return
if (
not isinstance(layout, dict)
or set(layout) != {"folder", "by_core"}
or not isinstance(layout["by_core"], bool)
or len(_safe_relative_path(layout["folder"], "save_directory.folder").parts) != 1
):
raise ValueError("invalid save_directory")
if layout["by_core"]:
# The core's folder heads each dest it names: dropped, never guessed,
# where the frontend does not sort saves by core.
for entry in entries:
folder = entry.get("core_folder")
dest = entry.get("dest")
if folder and isinstance(dest, str) and dest.split("/", 1)[0] != folder:
raise ValueError(f"{dest} does not start with its core_folder {folder}")
def _check_tags(entry: dict, field: str, where: str) -> None:
"""A selection field is absent or a short list of short strings."""
value = entry.get(field)
@@ -2531,6 +2628,7 @@ def _check_entry(
if dest in seen_destinations:
raise ValueError(f"duplicate manifest destination: {dest}")
seen_destinations.add(dest)
_check_root(entry, dest)
size = _check_download(entry, where, dest)
largest = size
@@ -2643,6 +2741,7 @@ def _validate_manifest(data: object, plat: str) -> dict:
raise ValueError("invalid data_directories list")
for index, entry in enumerate(data_dirs):
_check_data_directory(entry, f"data_directories[{index}]")
_check_save_directory(data)
declared_total_files = data.get("total_files")
if declared_total_files is not None and declared_total_files != len(files):
@@ -2667,6 +2766,7 @@ def _validate_manifest(data: object, plat: str) -> dict:
raise ValueError(f"duplicate or conflicting omitted destination: {dest}")
seen_omitted.add(dest)
_check_omission(entry, dest)
_check_root(entry, dest)
claims = entry.get("alternatives", [])
if not isinstance(claims, list) or len(claims) > MAX_SELECTION_TAGS:
raise ValueError(f"invalid omitted alternatives for {dest}")
@@ -3371,6 +3471,89 @@ def _sha256_file(path: Path) -> str:
return _digest_file(path, ("sha256",))["sha256"]
def _cfg_true(value: str | None) -> bool:
return (value or "").strip().lower() == "true"
def save_directory_for(
bios_path: Path, manifest: dict, override: Path | None = None,
) -> SavePlace | str:
"""The save directory the platform's frontend hands its cores, or why
this run cannot tell.
A RetroArch found by its retroarch.cfg answers from it: savefile_directory
as RetroArch expands it, a value left at default or naming no folder
being saves/ beside the config (configuration.c config_load_file keeps
the default unless path_is_directory), and sort_savefiles_enable, true by
default (config.def.h:1099). savefiles_in_content_dir and
sort_savefiles_by_content_enable put the directory beside each game: no
single place then. Without a config, the BIOS folder standing at its
default name puts the save directory beside it, as the pack lays them out.
``override`` (--save-dir) names the directory itself.
"""
layout = manifest.get("save_directory")
if not layout:
return "the file list names no save directory for this platform"
folder = layout["folder"]
listed = bool(layout["by_core"])
by_core = listed
found = _RETROARCH_CONFIGS.get(bios_path)
if found is not None:
cfg_path, app_dir = found
for key in ("savefiles_in_content_dir", "sort_savefiles_by_content_enable"):
if _cfg_true(_android_cfg_value(cfg_path, key)):
return (f"RetroArch keeps saves beside each game ({key} in {cfg_path}): "
"name the folder with --save-dir")
sort = _android_cfg_value(cfg_path, "sort_savefiles_enable")
if sort is not None:
by_core = _cfg_true(sort)
if override is not None:
return SavePlace(override, by_core, listed)
directory = _retroarch_cfg_dir(cfg_path, "savefile_directory", app_dir, folder)
if directory is None or not directory.is_dir():
directory = app_dir / folder
return SavePlace(directory, by_core, listed)
if override is not None:
return SavePlace(override, by_core, listed)
base = manifest.get("base_destination") or ""
if base and bios_path.name == base:
return SavePlace(bios_path.parent / folder, by_core, listed)
return (f"{bios_path} is not the frontend's default BIOS folder: "
"name its save directory with --save-dir")
def place_save_entries(
files: list[dict], bios_path: Path, manifest: dict, override: Path | None,
display: str,
) -> tuple[list[dict], list[dict]]:
"""Find the save directory for the entries read below it.
Returns the files to install and, when the save directory is unknown,
the entries left out as omissions, said once with the reason.
"""
saved = [entry for entry in files if entry.get("root") == "save"]
if not saved:
return files, []
place = save_directory_for(bios_path, manifest, override)
if isinstance(place, SavePlace):
_SAVE_DIRS[bios_path] = place
where = f"{place.directory}{os.sep}<core>" if place.by_core else str(place.directory)
print(f" {_plural(len(saved), 'file')} read from {display}'s save directory "
f"go to {where}")
return files, []
print(f" {_plural(len(saved), 'file')} read from {display}'s save directory "
f"left out: {place}")
kept = [entry for entry in files if entry.get("root") != "save"]
unplaced = [
{"dest": entry["dest"], "name": PurePosixPath(entry["dest"]).name,
"system": "", "required": False, "reason": "no_save_directory",
"cores": entry.get("cores"), "root": "save",
**({"core_folder": entry["core_folder"]} if entry.get("core_folder") else {})}
for entry in saved
]
return kept, unplaced
def _classify_local(entry: dict, dest: Path) -> str:
"""Return 'up_to_date' or 'mismatched' for a file already on disk."""
expected = {
@@ -3400,7 +3583,7 @@ def check_local(
present: list[tuple[dict, Path]] = []
for f in files:
dest = _destination_path(bios_path, f["dest"])
dest = _entry_path(bios_path, f)
if dest.exists():
present.append((f, dest))
else:
@@ -3667,12 +3850,12 @@ def plan_kept(entries: list[dict], root: Path) -> dict[Path, list[int]]:
planned: dict[Path, list[int]] = {}
for entry in entries:
try:
dest = _destination_path(root, entry["dest"])
dest = _entry_path(root, entry)
info = os.lstat(dest)
except (OSError, ValueError):
continue
if stat.S_ISREG(info.st_mode):
run = KEPT.plan(_kept_slot(root, entry["dest"])[0], info.st_size)
run = KEPT.plan(_kept_slot(*_entry_place(root, entry))[0], info.st_size)
tally = planned.setdefault(run, [0, 0])
tally[0] += 1
tally[1] += info.st_size
@@ -3759,7 +3942,7 @@ def _download_one(
if _stop.is_set():
return str(f.get("dest", "?")), False, "stopped"
try:
dest = _destination_path(bios_path, f["dest"])
dest = _entry_path(bios_path, f)
except ValueError:
return str(f.get("dest", "?")), False, "unsafe destination"
try:
@@ -3882,7 +4065,7 @@ def _download_one(
_clear_read_only(dest)
# The file in place, a personal dump or another revision, is
# moved aside first: replaced, it was gone for good.
KEPT.keep(*_kept_slot(bios_path, f["dest"]), dest, expected, tmp_path)
KEPT.keep(*_kept_slot(*_entry_place(bios_path, f)), dest, expected, tmp_path)
if partial is not None and _NO_RENAME_WHILE_OPEN:
# Windows renames no open file. Elsewhere the lock is held
# until the copy has its final name.
@@ -4168,7 +4351,9 @@ def _folder_writable(path: Path) -> bool:
# Options that take a value, as a hint gives them back.
_VALUED_OPTIONS = ("--system", "--core", "--emulator", "--region", "--target", "--cores-dir")
_VALUED_OPTIONS = (
"--system", "--core", "--emulator", "--region", "--target", "--cores-dir", "--save-dir",
)
class Invocation:
@@ -4466,9 +4651,11 @@ def _absent_required(omitted: list[dict], bios_path: Path) -> list[dict]:
if not entry.get("required", True) or not dest:
continue
try:
placed = _destination_path(bios_path, dest).is_file()
placed = _entry_path(bios_path, {**entry, "dest": dest}).is_file()
except ValueError:
placed = False # a path climbing out of the folder is never one we read
# A path climbing out of the folder is never one we read, and one
# below a save directory this run does not know is not looked at.
placed = False
if not placed:
absent.append(entry)
return absent
@@ -5720,6 +5907,14 @@ def main() -> None:
help="install only what the cores already installed read "
"(RetroArch, Lakka, RetroPie)",
)
parser.add_argument(
"--save-dir",
type=Path,
metavar="PATH",
help="the frontend's save directory (RetroArch: savefile_directory), "
"for the files some cores read there (Azahar's keys and system "
"data), when it is not found",
)
parser.add_argument(
"--cores-dir",
type=Path,
@@ -5840,7 +6035,8 @@ def main() -> None:
# The folder says where each file goes: nothing else applies.
alone = ("platform", "dest", "target", "every_board", "check", "list_platforms",
"list_targets", "standalone_copies", "emulator", "installed_cores",
"cores_dir", "system", "core", "region", "no_game_data", "list_systems",
"cores_dir", "save_dir", "system", "core", "region", "no_game_data",
"list_systems",
"list_cores", "romm_firmware", "list_emulators", "events", "quiet",
"yes", "verbose")
given = [name for name in alone if getattr(args, name)]
@@ -6434,6 +6630,13 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
f"under {library}")
bios_path = library
files, unplaced = place_save_entries(
files, bios_path, manifest, args.save_dir, display_name(plat_name, manifest)
)
if unplaced:
total_omitted += len(unplaced)
omitted_seen.extend(unplaced)
print("\nChecking existing files...")
to_download, up_to_date, mismatched = check_local(
files, bios_path, jobs=args.jobs, term=term
@@ -6448,7 +6651,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
# is in place: it goes.
for entry in up_to_date:
try:
stale = _destination_path(bios_path, entry["dest"])
stale = _entry_path(bios_path, entry)
except ValueError:
continue
stale = stale.parent / f".{stale.name}.part"
@@ -6460,9 +6663,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
narrowed = bool(
target or args.system or args.core or args.region or args.installed_cores
or len(files) + game_data_cut != len(manifest.get("files", [])) + len(
variants if case_kept else []
)
or len(files) + game_data_cut + len(unplaced)
!= len(manifest.get("files", [])) + len(variants if case_kept else [])
)
data_dirs = select_data_directories(manifest, files, narrowed)
data_states = [
+1 -1
View File
@@ -18,7 +18,7 @@ esac
TEMP_INSTALLER=""
TEMP_DIRECTORY=""
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
DEFAULT_INSTALL_SHA256="abc5b672e1d4b2f507b6d8c3d5c23a120a7dffea6e583559bdcc92c9cbbc7692"
DEFAULT_INSTALL_SHA256="b574c5beb84e9988a53a742f1abf0ad98fcb23dae961facef98c955856380704"
MAX_INSTALLER_BYTES=2097152
cleanup() {
+553
View File
@@ -0,0 +1,553 @@
"""Files a libretro core reads below the frontend's save directory.
Azahar's libretro core moves its user folder to <save_dir>/Azahar/
(src/citra_libretro/core_settings.cpp:1048-1080), so keys.txt, the shared
font and the NAND files are read there and not below the system directory.
The profile says so with ``load_from: save_dir``. The pack places these
files in the platform's save folder, the manifest marks them ``root: save``,
verify credits the same paths, and the installer writes them below the save
directory the frontend's config names.
"""
from __future__ import annotations
import functools
import hashlib
import http.server
import importlib.util
import json
import os
import shutil
import subprocess
import sys
import tempfile
import threading
import unittest
import zipfile
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import yaml # noqa: E402
from packpaths import ( # noqa: E402
EMULATOR_PACK_SAVE_FOLDER,
SaveLayout,
_resolve_destination,
save_destination,
save_layout,
)
from validation import ( # noqa: E402
entry_root,
outside_gap_scope,
read_from_save_dir,
read_from_system_dir,
)
_spec = importlib.util.spec_from_file_location("install", REPO_ROOT / "install.py")
install = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(install)
def _scratch(test: unittest.TestCase) -> Path:
"""A folder under the repository's tmp/, removed after the test."""
base = REPO_ROOT / "tmp"
base.mkdir(exist_ok=True)
path = Path(tempfile.mkdtemp(dir=str(base)))
test.addCleanup(shutil.rmtree, path, True)
return path
class TheEntryNamesItsRoot(unittest.TestCase):
ENTRY = {"name": "keys.txt", "path": "CoreX/sysdata/keys.txt",
"standalone_path": "sysdata/keys.txt",
"load_from": {"libretro": "save_dir"}}
def test_the_libretro_build_reads_below_the_save_directory(self):
self.assertTrue(read_from_save_dir(self.ENTRY))
self.assertEqual(entry_root(self.ENTRY), "save")
self.assertFalse(read_from_system_dir(self.ENTRY))
def test_the_standalone_build_has_no_frontend_save_directory(self):
self.assertFalse(read_from_save_dir(self.ENTRY, standalone=True))
self.assertEqual(entry_root(self.ENTRY, standalone=True), "system")
plain = {"name": "a.bin", "path": "a.bin", "load_from": "save_dir"}
self.assertTrue(read_from_save_dir(plain))
self.assertFalse(read_from_save_dir(plain, standalone=True))
def test_save_entries_are_in_the_pack_scope(self):
self.assertFalse(outside_gap_scope(self.ENTRY, False))
for other in ("content_dir", "working_directory", "internal_storage"):
with self.subTest(load_from=other):
self.assertTrue(outside_gap_scope({**self.ENTRY, "load_from": other}, False))
def test_an_emulator_pack_keeps_them_in_their_own_folder(self):
self.assertEqual(
_resolve_destination(self.ENTRY, False),
f"{EMULATOR_PACK_SAVE_FOLDER}/CoreX/sysdata/keys.txt",
)
self.assertEqual(_resolve_destination(self.ENTRY, True), "sysdata/keys.txt")
class ThePlatformNamesItsSaveDirectory(unittest.TestCase):
def test_the_registry_block(self):
self.assertEqual(
save_layout({"save_directory": {"folder": "saves", "by_core": True}}),
SaveLayout("saves", True),
)
for block in (None, {}, {"folder": ""}, {"folder": "a/b", "by_core": True}):
with self.subTest(block=block):
self.assertIsNone(save_layout({"save_directory": block}))
self.assertIsNone(save_layout({}))
def test_a_frontend_sorting_by_core_inserts_the_library_name(self):
layout = SaveLayout("saves", True)
self.assertEqual(save_destination("Azahar/sysdata/keys.txt", "Azahar", layout),
"saves/Azahar/Azahar/sysdata/keys.txt")
# Unknown: nowhere to put it. Empty: the core reads the same place
# sorted or not (an empty library_name, Cemu's own folder).
self.assertIsNone(save_destination("Azahar/sysdata/keys.txt", None, layout))
self.assertEqual(save_destination("Cemu/mlc01/x.ttf", "", layout),
"saves/Cemu/mlc01/x.ttf")
self.assertEqual(
save_destination("Azahar/sysdata/keys.txt", "", SaveLayout("savefiles", False)),
"savefiles/Azahar/sysdata/keys.txt",
)
def test_retroarch_and_lakka_have_one_retropie_none(self):
from common import load_platform_config
platforms = str(REPO_ROOT / "platforms")
self.assertEqual(save_layout(load_platform_config("retroarch", platforms)),
SaveLayout("saves", True))
self.assertEqual(save_layout(load_platform_config("lakka", platforms)),
SaveLayout("savefiles", True))
# RetroPie inherits RetroArch but keeps saves beside each game.
self.assertIsNone(save_layout(load_platform_config("retropie", platforms)))
class _Fixture(unittest.TestCase):
"""A RetroArch-like platform with a save directory, one without, and a
core reading one file below the system directory and one below the save
directory. The platform declares a keys.txt of its own in the BIOS tree:
it does not answer for the one the core reads below the save directory."""
SYS = b"SYSTEM FILE"
KEYS = b"SAVE DIR KEYS"
OTHER_KEYS = b"PLATFORM KEYS"
def setUp(self) -> None:
from common import _emulator_profiles_cache, _platform_config_cache
_emulator_profiles_cache.clear()
_platform_config_cache.clear()
self.addCleanup(_platform_config_cache.clear)
self.addCleanup(_emulator_profiles_cache.clear)
self.root = _scratch(self)
self.bios = self.root / "bios"
self.platforms = self.root / "platforms"
self.emulators = self.root / "emulators"
for folder in (self.bios / "X", self.bios / "Y", self.platforms, self.emulators):
folder.mkdir(parents=True)
files: dict[str, dict] = {}
by_name: dict[str, list[str]] = {}
by_md5: dict[str, str] = {}
for rel, data in (("X/sys.bin", self.SYS), ("X/keys.txt", self.KEYS),
("Y/keys.txt", self.OTHER_KEYS)):
path = self.bios / rel
path.write_bytes(data)
sha1 = hashlib.sha1(data).hexdigest()
md5 = hashlib.md5(data).hexdigest()
files[sha1] = {"path": str(path), "name": path.name, "md5": md5,
"size": len(data), "crc32": ""}
by_name.setdefault(path.name, []).append(sha1)
by_md5[md5] = sha1
self.db = {"files": files, "indexes": {
"by_md5": by_md5, "by_name": by_name, "by_crc32": {}, "by_path_suffix": {},
}}
system = {"files": [{"name": "keys.txt", "destination": "keys.txt",
"sha1": hashlib.sha1(self.OTHER_KEYS).hexdigest(),
"required": False}]}
for name in ("testra", "testnosave"):
(self.platforms / f"{name}.yml").write_text(yaml.safe_dump({
"platform": name, "verification_mode": "existence",
"base_destination": "system", "cores": ["corex"],
"systems": {"test-console": system},
}))
(self.platforms / "_registry.yml").write_text(yaml.safe_dump({"platforms": {
"testra": {"save_directory": {"folder": "saves", "by_core": True}},
"testnosave": {"save_directory": None},
}}))
(self.emulators / "corex.yml").write_text(yaml.safe_dump({
"emulator": "CoreX", "type": "libretro", "library_name": "CoreX",
"systems": ["test-console"],
"files": [
{"name": "sys.bin", "path": "sys.bin", "required": True,
"sha1": hashlib.sha1(self.SYS).hexdigest()},
{"name": "keys.txt", "path": "CoreX/sysdata/keys.txt",
"load_from": "save_dir", "required": False,
"sha1": hashlib.sha1(self.KEYS).hexdigest()},
],
}))
def config(self, name: str) -> dict:
from common import load_platform_config
return load_platform_config(name, str(self.platforms))
def profiles(self) -> dict:
from common import load_emulator_profiles
return load_emulator_profiles(str(self.emulators))
def pack(self, name: str) -> list[str]:
from generate_pack import generate_pack
out = self.root / f"out-{name}"
out.mkdir()
zip_path = generate_pack(
name, str(self.platforms), self.db, str(self.bios), str(out),
emulators_dir=str(self.emulators), offline=True,
)
with zipfile.ZipFile(zip_path) as zf:
self.readme = zf.read("README.txt").decode()
return sorted(zf.namelist())
def manifest(self, name: str) -> dict:
from generate_pack import generate_manifest
return generate_manifest(
name, str(self.platforms), self.db, str(self.bios),
str(self.platforms / "_registry.yml"),
emulators_dir=str(self.emulators), offline=True,
)
def verify(self, name: str) -> dict:
from verify import verify_platform
return verify_platform(self.config(name), self.db, str(self.emulators))
class ThePackPlacesThemInTheSaveFolder(_Fixture):
SAVED = "saves/CoreX/CoreX/sysdata/keys.txt"
def test_the_report_marks_the_entry(self):
from verify import find_undeclared_files
report = find_undeclared_files(
self.config("testra"), str(self.emulators), self.db, self.profiles()
)
saved = [u for u in report if u["name"] == "keys.txt"]
self.assertEqual(len(saved), 1, "the platform's keys.txt answers nothing here")
self.assertEqual(
{k: saved[0].get(k) for k in ("root", "core_folder", "path", "in_repo")},
{"root": "save", "core_folder": "CoreX",
"path": "CoreX/sysdata/keys.txt", "in_repo": True},
)
def test_pack_manifest_and_verify_agree(self):
names = self.pack("testra")
self.assertIn(self.SAVED, names)
self.assertIn("keys.txt", names, "the platform's own keys.txt stays")
self.assertIn("Files read from the save directory", self.readme)
report = self.verify("testra")
written = {n for n in names if n != "README.txt" and not n.endswith("/")}
self.assertEqual(set(report["pack_destinations"]), written)
manifest = self.manifest("testra")
saved = [e for e in manifest["files"] if e.get("root") == "save"]
self.assertEqual(len(saved), 1)
self.assertEqual(
{k: saved[0][k] for k in ("dest", "root", "core_folder", "sha1")},
{"dest": "CoreX/CoreX/sysdata/keys.txt", "root": "save", "core_folder": "CoreX",
"sha1": hashlib.sha1(self.KEYS).hexdigest()},
)
self.assertEqual(manifest["save_directory"], {"folder": "saves", "by_core": True})
# The fixture's copies live outside a repository checkout: give them
# the repository path the installer downloads from.
for entry in manifest["files"]:
entry["repo_path"] = f"bios/{Path(entry['repo_path']).name}"
install._validate_manifest(manifest, "testra")
def test_a_platform_without_a_save_directory_says_so(self):
names = self.pack("testnosave")
self.assertFalse(any(n.startswith("saves/") for n in names))
self.assertNotIn("Files read from the save directory", self.readme)
manifest = self.manifest("testnosave")
self.assertNotIn("save_directory", manifest)
omitted = [e for e in manifest["omitted_files"] if e.get("root") == "save"]
self.assertEqual([(e["dest"], e["reason"]) for e in omitted],
[("CoreX/CoreX/sysdata/keys.txt", "no_save_directory")])
report = self.verify("testnosave")
entry = next(u for u in report["undeclared_files"] if u.get("root") == "save")
self.assertFalse(entry["in_pack"])
self.assertIn("no fixed place", entry["omitted"])
self.assertFalse(any(p.startswith("saves/") for p in report["pack_destinations"]))
def test_the_emulator_pack_and_verify_emulator_agree(self):
from generate_pack import PackRecord, generate_emulator_pack
from verify import verify_emulator
record = PackRecord()
generate_emulator_pack(
["corex"], str(self.emulators), self.db, str(self.bios), str(self.root),
offline=True, platforms_dir=str(self.platforms), record=record,
)
self.assertIn("saves/CoreX/sysdata/keys.txt", record.files)
report = verify_emulator(["corex"], str(self.emulators), self.db,
platforms_dir=str(self.platforms))
self.assertEqual(set(report["pack_destinations"]), set(record.files))
class TheInstallerFindsTheSaveDirectory(unittest.TestCase):
MANIFEST = {"base_destination": "system",
"save_directory": {"folder": "saves", "by_core": True}}
def setUp(self) -> None:
self.root = _scratch(self)
self.app = self.root / "RetroArch"
self.bios = self.app / "system"
self.cfg = self.app / "retroarch.cfg"
self.bios.mkdir(parents=True)
self.addCleanup(install._RETROARCH_CONFIGS.pop, self.bios, None)
self.addCleanup(install._SAVE_DIRS.pop, self.bios, None)
def place(self, cfg: str | None, override: Path | None = None):
if cfg is not None:
self.cfg.write_text(cfg)
install._RETROARCH_CONFIGS[self.bios] = (self.cfg, self.app)
return install.save_directory_for(self.bios, self.MANIFEST, override)
def test_a_config_left_at_its_defaults(self):
self.assertEqual(self.place('system_directory = ":\\\\system"\n'),
install.SavePlace(self.app / "saves", True))
def test_a_configured_folder_and_sorting(self):
saves = self.root / "elsewhere"
saves.mkdir()
cfg = f'savefile_directory = "{saves}"\nsort_savefiles_enable = "false"\n'
self.assertEqual(self.place(cfg), install.SavePlace(saves, False))
def test_a_configured_folder_that_does_not_exist_is_ignored(self):
cfg = f'savefile_directory = "{self.root / "gone"}"\n'
self.assertEqual(self.place(cfg), install.SavePlace(self.app / "saves", True))
def test_saves_beside_each_game_have_no_single_place(self):
for key in ("savefiles_in_content_dir", "sort_savefiles_by_content_enable"):
with self.subTest(key=key):
found = self.place(f'{key} = "true"\n')
self.assertIsInstance(found, str)
self.assertIn(key, found)
def test_without_a_config(self):
self.assertEqual(self.place(None), install.SavePlace(self.app / "saves", True))
other = self.root / "BIOS"
found = install.save_directory_for(other, self.MANIFEST)
self.assertIsInstance(found, str)
self.assertIn("--save-dir", found)
self.assertEqual(install.save_directory_for(other, self.MANIFEST, self.root / "s"),
install.SavePlace(self.root / "s", True))
def test_the_entry_path(self):
saves = (self.app / "saves").resolve()
install._SAVE_DIRS[self.bios] = install.SavePlace(self.app / "saves", True)
entry = {"dest": "Azahar/Azahar/sysdata/keys.txt", "root": "save",
"core_folder": "Azahar"}
self.assertEqual(install._entry_path(self.bios, entry),
saves / "Azahar/Azahar/sysdata/keys.txt")
# sort_savefiles_enable off: the core is handed the save directory.
install._SAVE_DIRS[self.bios] = install.SavePlace(self.app / "saves", False)
self.assertEqual(install._entry_path(self.bios, entry),
saves / "Azahar/sysdata/keys.txt")
# A list laid out unsorted, a frontend sorting: the folder goes in.
install._SAVE_DIRS[self.bios] = install.SavePlace(self.app / "saves", True, False)
self.assertEqual(
install._entry_path(self.bios, {**entry, "dest": "Azahar/sysdata/keys.txt"}),
saves / "Azahar/Azahar/sysdata/keys.txt",
)
# Cemu adds its folder once: no core_folder, the same place either way.
cemu = {"dest": "Cemu/mlc01/a.ttf", "root": "save"}
for by_core in (True, False):
install._SAVE_DIRS[self.bios] = install.SavePlace(self.app / "saves", by_core)
self.assertEqual(install._entry_path(self.bios, cemu), saves / "Cemu/mlc01/a.ttf")
plain = {"dest": "Azahar/sysdata/keys.txt"}
self.assertEqual(install._entry_path(self.bios, plain),
self.bios.resolve() / "Azahar/sysdata/keys.txt")
class TheManifestBoundary(unittest.TestCase):
ENTRY = {"dest": "Azahar/Azahar/sysdata/keys.txt", "sha1": "a" * 40, "sha256": "b" * 64,
"size": 1, "repo_path": "bios/x/keys.txt", "cores": ["azahar"],
"root": "save", "core_folder": "Azahar"}
def manifest(self, entry: dict, layout: object = ...) -> dict:
data = {"manifest_version": 2, "platform": "retroarch", "files": [entry],
"total_files": 1, "total_size": 1}
if layout is not ...:
data["save_directory"] = layout
return data
def test_a_save_entry_crosses_it(self):
install._validate_manifest(
self.manifest(self.ENTRY, {"folder": "saves", "by_core": True}), "retroarch"
)
def test_what_is_refused(self):
layout = {"folder": "saves", "by_core": True}
for label, data in (
("no save_directory", self.manifest(self.ENTRY)),
("other root", self.manifest({**self.ENTRY, "root": "content"}, layout)),
("climbing core folder", self.manifest({**self.ENTRY, "core_folder": ".."}, layout)),
("nested core folder", self.manifest({**self.ENTRY, "core_folder": "a/b"}, layout)),
("core folder on a BIOS file", self.manifest(
{k: v for k, v in self.ENTRY.items() if k != "root"}, layout)),
("climbing save folder", self.manifest(self.ENTRY, {"folder": "..", "by_core": True})),
("save folder with a path", self.manifest(self.ENTRY, {"folder": "a/b", "by_core": True})),
("by_core not a flag", self.manifest(self.ENTRY, {"folder": "saves", "by_core": 1})),
("dest without its core folder", self.manifest(
{**self.ENTRY, "dest": "sysdata/keys.txt"}, layout)),
):
with self.subTest(label), self.assertRaises(ValueError):
install._validate_manifest(data, "retroarch")
class _Repository:
"""A loopback copy of the repository serving one manifest and its files."""
def __init__(self, test: unittest.TestCase, files: dict[str, tuple[bytes, dict]]):
self.root = _scratch(test)
(self.root / "install").mkdir()
(self.root / "bios").mkdir()
entries = []
for dest, (payload, extra) in files.items():
stored = dest.replace("/", "_")
(self.root / "bios" / stored).write_bytes(payload)
entries.append({
"dest": dest, "sha1": hashlib.sha1(payload).hexdigest(),
"sha256": hashlib.sha256(payload).hexdigest(), "size": len(payload),
"repo_path": f"bios/{stored}", "cores": None, **extra,
})
manifest = {"manifest_version": 2, "platform": "retroarch",
"base_destination": "system", "files": entries,
"save_directory": {"folder": "saves", "by_core": True}}
(self.root / "install" / "retroarch.json").write_text(json.dumps(manifest))
handler = functools.partial(_Quiet, directory=str(self.root))
self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
threading.Thread(target=self.httpd.serve_forever, daemon=True).start()
test.addCleanup(self.httpd.server_close)
test.addCleanup(self.httpd.shutdown)
def run(self, *options: str) -> subprocess.CompletedProcess:
env = dict(os.environ)
env["RETROBIOS_BASE_URL"] = f"http://127.0.0.1:{self.httpd.server_address[1]}"
return subprocess.run(
[sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch",
"--yes", *options],
env=env, capture_output=True, text=True, timeout=120, check=False,
stdin=subprocess.DEVNULL,
)
class _Quiet(http.server.SimpleHTTPRequestHandler):
def log_message(self, *args) -> None: # noqa: D102
pass
class TheInstallerWritesThem(unittest.TestCase):
FILES = {
"sys.bin": (b"system file", {}),
"Azahar/Azahar/sysdata/keys.txt": (
b"keys below the save directory", {"root": "save", "core_folder": "Azahar"},
),
}
def test_beside_the_default_bios_folder(self):
repo = _Repository(self, self.FILES)
app = _scratch(self)
proc = repo.run("--dest", str(app / "system"))
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertEqual((app / "system" / "sys.bin").read_bytes(), b"system file")
self.assertEqual(
(app / "saves" / "Azahar" / "Azahar" / "sysdata" / "keys.txt").read_bytes(),
b"keys below the save directory",
)
self.assertFalse((app / "system" / "Azahar").exists())
self.assertIn("read from RetroArch's save directory go to", proc.stdout)
def test_where_save_dir_points(self):
repo = _Repository(self, self.FILES)
app = _scratch(self)
saves = app / "my saves"
proc = repo.run("--dest", str(app / "BIOS"), "--save-dir", str(saves))
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertTrue((saves / "Azahar" / "Azahar" / "sysdata" / "keys.txt").is_file())
def test_left_out_and_said_when_the_save_directory_is_unknown(self):
repo = _Repository(self, self.FILES)
app = _scratch(self)
proc = repo.run("--dest", str(app / "BIOS"))
self.assertTrue((app / "BIOS" / "sys.bin").is_file(), proc.stdout + proc.stderr)
self.assertFalse(any(app.rglob("keys.txt")))
self.assertIn("--save-dir", proc.stdout)
class TheProfilesFollowTheContract(unittest.TestCase):
def test_every_save_dir_entry_has_a_library_name(self):
try:
from validate_schemas import _save_dir_without_library_name
except ImportError:
self.skipTest("validate_schemas needs jsonschema")
for path in sorted((REPO_ROOT / "emulators").glob("*.yml")):
document = yaml.safe_load(path.read_text(encoding="utf-8"))
with self.subTest(profile=path.name):
self.assertEqual(_save_dir_without_library_name(path, document), [])
def test_azahar_reads_its_user_folder_below_the_save_directory(self):
profile = yaml.safe_load((REPO_ROOT / "emulators" / "azahar.yml").read_text())
self.assertEqual(profile["library_name"], "Azahar")
for entry in profile["files"]:
with self.subTest(entry=entry["name"]):
self.assertTrue(read_from_save_dir(entry))
self.assertTrue(entry["path"].startswith("Azahar/"))
def test_the_registry_blocks(self):
try:
from validate_schemas import _save_directory_blocks
except ImportError:
self.skipTest("validate_schemas needs jsonschema")
self.assertEqual(_save_directory_blocks(), [])
def test_the_published_retroarch_list_places_azahar(self):
manifest = json.loads((REPO_ROOT / "install" / "retroarch.json").read_text())
self.assertEqual(manifest.get("save_directory"), {"folder": "saves", "by_core": True})
azahar = {e["dest"]: e for e in manifest["files"] if e.get("core_folder") == "Azahar"}
self.assertIn("Azahar/Azahar/sysdata/keys.txt", azahar)
self.assertTrue(all(e["root"] == "save" for e in azahar.values()))
class TheSchemasCarryTheFields(unittest.TestCase):
def test_install_manifest_and_profile_schemas(self):
try:
from jsonschema import Draft202012Validator
except ImportError:
self.skipTest("jsonschema is not installed")
schema = json.loads((REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text())
validator = Draft202012Validator(schema)
manifest = json.loads((REPO_ROOT / "install" / "retroarch.json").read_text())
self.assertEqual(list(validator.iter_errors(manifest)), [])
bad = {**manifest, "files": [{**manifest["files"][0], "root": "content"}]}
self.assertTrue(list(validator.iter_errors(bad)))
emulator = json.loads((REPO_ROOT / "schemas" / "emulator.schema.json").read_text())
profile = yaml.safe_load((REPO_ROOT / "emulators" / "azahar.yml").read_text())
self.assertEqual(list(Draft202012Validator(emulator).iter_errors(profile)), [])
self.assertTrue(list(Draft202012Validator(emulator).iter_errors(
{**profile, "library_name": "a/b"}
)))
if __name__ == "__main__":
unittest.main()