kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-06 10:03:49 -06:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge installer fixes, fourth pass

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-11-06 10:28:14 +01:00
2 parents e491ff1bc6 333f1ac3ca
commit a29b836730
27 files changed
+888 -158
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

install.ps1
+1 -1
View File
@@ -16,7 +16,7 @@
# and Invoke-WebRequest throws on a cut transfer before the hash is read:
# the site's build rewrites this line there (scripts/sitebootstrap.py).
$mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that."
$defaultInstallSha256 = "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858"
$defaultInstallSha256 = "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1"
$maximumInstallerBytes = 2MB
$installerExit = 0
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
install.py
+215 -35
View File
@@ -291,7 +291,6 @@ def _os_default_dests(os_type: str) -> dict[str, Path]:
the command line but not found must not land in ~/bios, which none of
them reads.
"""
home = Path.home()
if os_type == "windows":
return {
# RetroArch's system directory defaults to <exe folder>\system
@@ -307,13 +306,13 @@ def _os_default_dests(os_type: str) -> dict[str, Path]:
# RetroArch may be the Linux one inside the distribution, while RetroBat
# only exists on the Windows side, reached through /mnt/c.
return {
"retroarch": home / ".config" / "retroarch" / "system",
"retroarch": _unix_retroarch_base() / "system",
"retrobat": _host_path("C:/RetroBat/bios"),
}
if os_type == "darwin":
return {"retroarch": _macos_retroarch_base() / "system"}
return {
"retroarch": home / ".config" / "retroarch" / "system",
"retroarch": _unix_retroarch_base() / "system",
}
@@ -681,8 +680,9 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = {
# found nowhere and no folder to assume) or from (a folder named with --from,
# reason: not_found, not_a_folder, unreadable, or system for the root
# of the system and /proc, /sys, /dev) or dest (reason: unreadable, a
# folder above it this user cannot open, or not_writable, the BIOS
# folder cannot be created, a summary following) or option (reason:
# folder above it this user cannot open, not_a_folder, the BIOS folder
# is a file or lies under one, or not_writable, the BIOS folder cannot
# be created, a summary following) or option (reason:
# unknown, a platform, target, system, core or emulator the run cannot
# use, refused before any download).
"error": ("stage", "reason", "platform", "message"),
@@ -1385,6 +1385,17 @@ def _macos_retroarch_base() -> Path:
return Path.home() / "Documents" / "RetroArch"
def _unix_retroarch_base() -> Path:
"""RetroArch's base folder on Linux: $XDG_CONFIG_HOME/retroarch when the
variable is set, else ~/.config/retroarch (fill_pathname_application_data
in file_path_special.c, frontend_unix_get_env in platform_unix.c). The
code tests only that the variable exists, so an empty one counts."""
xdg = os.environ.get("XDG_CONFIG_HOME")
if xdg is not None:
return Path(xdg) / "retroarch"
return Path.home() / ".config" / "retroarch"
# System directory -> the retroarch.cfg it was read from and the directory
# ':' expands to there, for finding the cores of the same installation.
_RETROARCH_CONFIGS: dict[Path, tuple[Path, Path]] = {}
@@ -2773,11 +2784,16 @@ def detect_platforms(os_type: str) -> list[tuple[str, Path]]:
if ra_dir:
found.append(("retroarch", ra_dir))
# RetroArch native
native_cfg = home / ".config" / "retroarch" / "retroarch.cfg"
ra_dir = _parse_retroarch_system_dir(native_cfg)
if ra_dir:
found.append(("retroarch", ra_dir))
# RetroArch native: the config under its base folder, then
# ~/.retroarch.cfg when that one is missing (open_default_config_file
# in configuration.c). Either way the system folder defaults to the
# base folder's.
native_base = _unix_retroarch_base()
for native_cfg in (native_base / "retroarch.cfg", home / ".retroarch.cfg"):
ra_dir = _parse_retroarch_system_dir(native_cfg, native_base)
if ra_dir:
found.append(("retroarch", ra_dir))
break
if os_type == "darwin":
# open_default_config_file reads <Application Support>/RetroArch/config
@@ -3692,6 +3708,9 @@ def _check_data_directory(entry: object, where: str) -> None:
strip = entry.get("strip_components", 0)
if not isinstance(strip, int) or isinstance(strip, bool) or not 0 <= strip <= 16:
raise ValueError(f"{where}.strip_components is invalid")
size = entry.get("archive_size", 1)
if not isinstance(size, int) or isinstance(size, bool) or size < 1:
raise ValueError(f"{where}.archive_size is invalid")
if kind == "tarball" and (
not isinstance(entry.get("source_path"), str) or not entry["source_path"]
):
@@ -6440,7 +6459,8 @@ def _report_outcome(
kept = _kept_lines() + ([
f"{_plural(len(unplaced), 'file')} read from "
f"{unplaced[0].get('frontend') or 'the frontend'}'s save directory left out: "
f"its place is not known. {LINE_SAVE_REMEDY}"] if unplaced else [])
+ (f"{unplaced[0]['save_why']}." if unplaced[0].get("save_why")
else f"its place is not known. {LINE_SAVE_REMEDY}")] if unplaced else [])
kept += list(notes or [])
if interrupted:
say(f"\nStopped. {_plural(downloaded, 'file')} installed.")
@@ -6856,11 +6876,12 @@ def _print_plan(
# files, and 2636 data files" read as two kinds of one thing. The kinds
# follow, adding up to it.
total = counts.files + tree_files
parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_counts.size)})"]
tree_bytes = sum(data_fetch_bytes(entry, needed) for entry, needed in trees)
parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_bytes)})"]
if total else [])
split = parts_words(counts if (manifest or {}).get("categorized")
else tally({"size": e.get("size")} for e in to_download),
tree_files, tree_counts.size)
tree_files, tree_bytes)
if split and trees:
split += f", the data files extracted from {_plural(len(trees), 'archive')}"
# A pack is copied from, not downloaded: "Would download 23 files" was
@@ -6935,8 +6956,11 @@ def _print_plan(
"every system and its files.")
for entry, needed in trees:
source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"]
fetched = data_fetch_bytes(entry, needed)
archive = ("" if fetched == sum(meta.get("size", 0) for meta in needed.values())
else f" (a {format_size(fetched)} archive)")
print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, "
f"from {source}")
f"from {source}{archive}")
for folder, (count, size) in sorted((kept or {}).items()):
print(f"\n Would keep {_plural(count, 'file')} you have ({format_size(size)}), "
f"whose contents differ, in {folder}")
@@ -7062,20 +7086,72 @@ def download_files(
return failed, interrupted
def _fetch_archive(url: str, directory: Path, progress: Progress | None = None) -> Path:
def _archive_validator(headers: object) -> str:
"""What names this revision of an archive to its server: an If-Range
with it gets the rest of the same bytes, or the whole new archive."""
if not headers:
return ""
etag = headers.get("ETag", "") or ""
# A weak tag never answers a range (RFC 9110 13.1.5).
return etag if etag and not etag.startswith("W/") else headers.get("Last-Modified", "") or ""
def _fetch_archive(url: str, directory: Path, progress: Progress | None = None,
keep: Path | None = None) -> Path:
"""A data directory's source archive, in *directory*, retried like files.
progress draws its bytes as they come, an attempt that failed taken back."""
progress draws its bytes as they come, an attempt that failed taken back.
With keep, the archive is written there and kept when the transfer is
cut, beside a note of its address and the validator its server sent:
the next attempt, or the next run, asks for the rest with Range and
If-Range, and a server that changed the archive sends it whole. A cut
50 MB archive started again from zero, and a link that drops more often
than one archive takes never installed the tree.
"""
_network_allowed(url)
note = keep.with_name(keep.name + ".source") if keep is not None else None
reason: Exception | None = None
for attempt in range(1, MAX_RETRIES + 1):
handle = tempfile.NamedTemporaryFile(dir=directory, suffix=".archive", delete=False)
offset = 0
validator = ""
if keep is not None:
try:
recorded = note.read_text(encoding="utf-8").split("\n")
if recorded[0] == url and len(recorded) > 1 and recorded[1]:
validator = recorded[1]
offset = keep.stat().st_size
except (OSError, UnicodeError):
offset = 0
request = urllib.request.Request(url)
if offset:
request.add_header("Range", f"bytes={offset}-")
request.add_header("If-Range", validator)
handle = (open(keep, "r+b" if offset else "wb") if keep is not None else
tempfile.NamedTemporaryFile(dir=directory, suffix=".archive", delete=False))
received = 0
resumed = 0
try:
with handle, urllib.request.urlopen(url, timeout=60) as resp:
with handle, urllib.request.urlopen(request, timeout=60) as resp:
headers = resp.headers or {}
ranged = (offset and getattr(resp, "status", 200) == 206 and str(
headers.get("Content-Range", "")).startswith(f"bytes {offset}-"))
if not ranged:
offset = 0
handle.seek(offset)
handle.truncate()
if keep is not None and not offset:
# Written before the first byte: a cut transfer is then
# resumed only against the revision it holds.
note.write_text(f"{url}\n{_archive_validator(headers)}\n",
encoding="utf-8")
if progress is not None:
length = resp.headers.get("Content-Length", "")
length = headers.get("Content-Length", "")
if length.isdigit():
progress.expect(int(length))
progress.expect(offset + int(length))
if offset:
progress.add(offset, resumed=True)
resumed = offset
total = offset
while True:
if _stop.is_set():
raise _Interrupted()
@@ -7083,30 +7159,51 @@ def _fetch_archive(url: str, directory: Path, progress: Progress | None = None)
if not chunk:
break
received += len(chunk)
if received > MAX_DATA_ARCHIVE_SIZE:
total += len(chunk)
if total > MAX_DATA_ARCHIVE_SIZE:
raise ValueError("archive exceeds the size limit")
handle.write(chunk)
if progress is not None:
progress.add(len(chunk))
return Path(handle.name)
return keep if keep is not None else Path(handle.name)
except urllib.error.HTTPError as exc:
Path(handle.name).unlink(missing_ok=True)
if exc.code < 500 and exc.code != 429:
if keep is None or exc.code == 416 or exc.code < 500 and exc.code != 429:
# Refused, or a range past an archive that shrank: no
# attempt resumes it.
_drop_archive(Path(handle.name), note)
if exc.code < 500 and exc.code not in (416, 429):
raise
reason = exc
except (urllib.error.URLError, OSError, http.client.HTTPException) as exc:
Path(handle.name).unlink(missing_ok=True)
if keep is None:
Path(handle.name).unlink(missing_ok=True)
reason = exc if isinstance(exc, OSError) else ConnectionError(repr(exc))
except BaseException:
Path(handle.name).unlink(missing_ok=True)
except ValueError:
_drop_archive(Path(handle.name), note)
raise
if progress is not None and received:
progress.add(-received)
except BaseException:
# Stopped: a kept archive waits for the next run.
if keep is None:
Path(handle.name).unlink(missing_ok=True)
raise
if progress is not None:
if received:
progress.add(-received)
if resumed:
progress.add(-resumed)
progress.forget_resumed(resumed)
if attempt < MAX_RETRIES:
time.sleep(_retry_delay(attempt, None))
raise reason if reason else OSError("download failed")
def _drop_archive(path: Path, note: Path | None) -> None:
"""Remove an archive and the note of where it came from."""
path.unlink(missing_ok=True)
if note is not None:
note.unlink(missing_ok=True)
def _archive_members(
archive: Path, entry: dict
) -> Iterator[tuple[str, int, Callable[[], BinaryIO]]]:
@@ -7167,6 +7264,19 @@ def data_directory_state(entry: dict, bios_path: Path,
return root, needed, present
def data_fetch_bytes(entry: dict, needed: dict[str, dict]) -> int:
"""What getting a tree's missing files costs: its source archive, which
comes whole for one file, or from a pack the files themselves. A list
that records no archive size gives the files' own sizes: a tarball whose
tree is 10 KB was announced as 10 KB and was a download of 21 MB."""
if not needed:
return 0
size = entry.get("archive_size")
if LOCAL.pack is None and isinstance(size, int) and size > 0:
return size
return sum(meta.get("size", 0) for meta in needed.values())
def install_data_directory(entry: dict, bios_path: Path,
state: tuple[Path, dict[str, dict], int] | None = None,
progress: Progress | None = None) -> dict:
@@ -7186,6 +7296,7 @@ def install_data_directory(entry: dict, bios_path: Path,
if not needed:
return counts
_writing.set()
fetched: Path | None = None
try:
root.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=root) as scratch:
@@ -7193,7 +7304,11 @@ def install_data_directory(entry: dict, bios_path: Path,
# The pack holds the tree where it extracts, already mapped.
members = LOCAL.pack.tree(entry.get("dest", ""))
else:
archive = _fetch_archive(entry["url"], Path(scratch), progress)
# Beside the tree, outside the scratch folder: a transfer
# cut short is resumed by the next attempt or the next run.
kept = root / f".{entry['ref']}.archive.part"
archive = _fetch_archive(entry["url"], Path(scratch), progress, kept)
fetched = archive
members = _archive_members(archive, entry)
for rel, declared, opener in members:
meta = needed.pop(rel, None)
@@ -7227,6 +7342,10 @@ def install_data_directory(entry: dict, bios_path: Path,
except _MEMBER_ERRORS as exc:
# A damaged, encrypted or unsupported member, in words.
counts["error"] = unreadable_words(exc)
finally:
if fetched is not None:
# Read whole: the next run fetches the archive afresh.
_drop_archive(fetched, fetched.with_name(fetched.name + ".source"))
counts["absent"] = len(needed)
return counts
@@ -8620,6 +8739,13 @@ def saves_line(display: str, count: int, where: str) -> str:
f"{shown_path(folder)}")
def unlisted_saves_words(display: str) -> str:
"""Why a file list holds none of the files read from the save
directory: the platform keeps it in no fixed place, and the list leaves
them out with no hash or address, which --save-dir cannot change."""
return f"{display} keeps it in no fixed place, and its file list holds none of them"
def unplaced_line(display: str, count: int, remedy: str) -> str:
"""The files read from an unknown save directory, left out."""
return (f"{_plural(count, 'file')} read from {display}'s save directory left out: "
@@ -8939,7 +9065,7 @@ def found_head(files: list[dict], trees: list[dict], bios_path: Path) -> str:
for tree in trees:
_root, wanted, _present = data_directory_state(tree, bios_path, remember=True)
data += len(wanted)
data_bytes += sum(meta.get("size", 0) for meta in wanted.values())
data_bytes += data_fetch_bytes(tree, wanted)
left = [entry for entry in needed if id(entry) not in copies]
rest = len(left) + data
where = ", ".join(f"{count} found in {_last_name(label)}"
@@ -13413,6 +13539,29 @@ def main() -> None:
if (args.list_platforms or args.list_targets or args.list_systems
or args.list_cores or args.list_emulators):
parser.error("--for-games narrows an install, not a listing")
# Applied or refused, never ignored: a run prints one list, and the
# narrowing options apply to --list-systems and --list-cores only, past
# --target and --installed-cores, which --list-emulators also reads.
listings = [flag for flag, given in (
("--list-platforms", args.list_platforms), ("--list-targets", args.list_targets),
("--list-systems", args.list_systems), ("--list-cores", args.list_cores),
("--list-emulators", args.list_emulators)) if given]
if len(listings) > 1:
parser.error(f"{listings[0]} and {listings[1]} each print a list: run them "
"one at a time")
if listings and listings[0] in ("--list-platforms", "--list-targets",
"--list-emulators"):
unread = (("--system", args.system), ("--core", args.core),
("--region", args.region), ("--emulator", args.emulator))
if listings[0] != "--list-emulators":
unread += (("--target", args.target), ("--every-board", args.every_board),
("--installed-cores", args.installed_cores))
if listings[0] == "--list-platforms":
unread += (("--platform", args.platform),)
given = next((flag for flag, value in unread if value), "")
if given:
parser.error(f"{listings[0]} prints the whole list, which {given} "
"does not narrow")
if args.declared_only:
# Applied or refused, never ignored.
for given, reason in (
@@ -13758,6 +13907,27 @@ def _refuse_closed(dest: Path, named: str) -> None:
sys.exit(1)
def _refuse_file(dest: Path, named: str) -> None:
"""End the run on a BIOS folder that is a file, or lies under one.
mkdir then failed on EEXIST or ENOTDIR, and the run blamed permissions
and offered sudo, which fails the same way.
"""
for folder in (dest, *dest.parents):
if _exists(folder):
break
else:
return
if _is_dir(folder):
return
where = (f"{shown_path(dest)} is a file, not a folder" if folder == dest
else f"{shown_path(dest)} cannot be a folder: {shown_path(folder)} is a file")
EVENTS.emit("error", stage="dest", reason="not_a_folder", platform="", message=where)
print(f"Error: {screen_text(where)}. Name the folder the BIOS files go in, not "
f"the file {named}.", file=sys.stderr)
sys.exit(1)
def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest",
again: bool = False) -> bool:
"""Say that a --dest folder does not exist before anything goes there.
@@ -13772,6 +13942,7 @@ def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest",
a card's folder ended the run where it meant "not that card".
"""
_refuse_closed(dest, named)
_refuse_file(dest, named)
if dest.exists():
return True
nearest = dest.absolute()
@@ -15099,6 +15270,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
else f"{found_in[0]} names" if found_in else "detected here")
# One setup by now: two found without questions stopped above.
_refuse_closed(path, f"{named} (name another with --dest)")
_refuse_file(path, f"{named} (name another with --dest)")
total_downloaded = 0
total_up_to_date = 0
@@ -15137,7 +15309,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
# --installed-cores is below: EmuDeck installed 3.2 GB and only then was
# steamos refused for RetroArch, with no summary of what had landed.
target_lists: dict[str, dict] = {}
if args.target and not args.list_targets:
if args.target:
for plat_name, _bios_path in platforms:
targets = fetch_targets(plat_name)
if targets is None:
@@ -15333,6 +15505,10 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
display = display_name(plat_name, manifest)
files, unplaced = place_save_entries(
files, bios_path, manifest, args.save_dir, display, say=False)
if not manifest.get("save_directory"):
# Its own list leaves these files out: naming the folder with
# --save-dir installs none of them, and was advised in a loop.
_SAVE_WHY[bios_path] = unlisted_saves_words(display)
# The games of the card, read before the choices: where it holds
# some, what they need is the menu's first choice, and --for-games
# narrows the list here, before the systems are counted.
@@ -15535,7 +15711,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
total_omitted += len(omitted_files)
# Named for the end of the run: ROCKNIX's own list leaves them out,
# and the end said "the frontend's save directory".
omitted_seen.extend({**entry, "frontend": display}
omitted_seen.extend({**entry, "frontend": display,
"save_why": unlisted_saves_words(display)}
if entry.get("reason") == "no_save_directory"
and not entry.get("frontend") else entry
for entry in omitted_files)
@@ -15608,8 +15785,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
plat_name, manifest, bios_path, files, up_to_date, mismatched,
sum(len(state[1]) for _entry, state in data_states), len(omitted_files),
chosen=selection[0] if selection else None,
data_bytes=sum(meta.get("size", 0) for _entry, state in data_states
for meta in state[1].values()),
data_bytes=sum(data_fetch_bytes(entry, state[1])
for entry, state in data_states),
)
emit_copies(plat_name, to_download, copies, bios_index, bios_path, files)
emit_unreadable(plat_name, unread)
@@ -21150,6 +21327,9 @@ def _unplaced_lines(summary: dict | None, state: TuiState,
if not count:
return []
name = display_name(state.platform) if state.platform else "the frontend"
if state.manifest and not state.manifest.get("save_directory"):
return [f"{_plural(count, 'file')} read from {name}'s save directory left out: "
f"{unlisted_saves_words(name)}."]
return [f"{_plural(count, 'file')} read from {name}'s save directory left out: "
f"its place is not known. {remedy or TUI_SAVE_REMEDY}"]
install.sh
+1 -1
View File
@@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins
# curl or wget stop on a cut transfer before the hash is read: the site's
# build rewrites this line there (scripts/sitebootstrap.py).
MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that."
DEFAULT_INSTALL_SHA256="34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858"
DEFAULT_INSTALL_SHA256="fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1"
MAX_INSTALLER_BYTES=2097152
cleanup() {
install/amberelec.json
+8 -4
View File
@@ -63489,7 +63489,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "fbneo-cheats",
@@ -77218,7 +77219,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -77451,7 +77453,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -77823,7 +77826,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
}
]
}
install/batocera.json
+4 -2
View File
@@ -225291,7 +225291,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -225524,7 +225525,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
}
],
"save_directory": {
install/emudeck.json
+18 -9
View File
@@ -100486,7 +100486,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "dinothawr",
@@ -100934,7 +100935,8 @@
"size": 105
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 5752250
},
{
"ref": "dirksimple",
@@ -100990,7 +100992,8 @@
"size": 12988
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 175384
},
{
"ref": "dolphin-sys",
@@ -109666,7 +109669,8 @@
"strip_components": 2,
"exclude": [
"Themes"
]
],
"archive_size": 3197914
},
{
"ref": "fbneo-cheats",
@@ -123395,7 +123399,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -123628,7 +123633,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -124000,7 +124006,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "lrps2",
@@ -124016,7 +124023,8 @@
"size": 1814032
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 318896
},
{
"ref": "yaps2-resources",
@@ -124476,7 +124484,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
],
"save_directory": {
install/emuelec.json
+10 -5
View File
@@ -69309,7 +69309,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "dinothawr",
@@ -69757,7 +69758,8 @@
"size": 105
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 5752250
},
{
"ref": "fbneo-cheats",
@@ -83486,7 +83488,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -83719,7 +83722,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -84091,7 +84095,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
}
]
}
install/knulli.json
+8 -4
View File
@@ -157933,7 +157933,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -158166,7 +158167,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "lrps2",
@@ -158182,7 +158184,8 @@
"size": 1814032
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 318896
},
{
"ref": "ppsspp-assets",
@@ -158870,7 +158873,8 @@
"size": 15005
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 9844388
}
],
"save_directory": {
install/lakka.json
+14 -7
View File
@@ -115932,7 +115932,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "cemu-gameprofiles",
@@ -116888,7 +116889,8 @@
"size": 36
}
},
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default"
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default",
"archive_size": 21360579
},
{
"ref": "dirksimple",
@@ -116944,7 +116946,8 @@
"size": 12988
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 175384
},
{
"ref": "fbneo-cheats",
@@ -130673,7 +130676,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -130906,7 +130910,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -131278,7 +131283,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "yaps2-resources",
@@ -131738,7 +131744,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
],
"save_directory": {
install/launchers/index.json
+8 -8
View File
@@ -27,8 +27,8 @@
{
"dest": "roms/ports/retrobios/install.py",
"repo_path": "install.py",
"size": 1103111,
"sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858",
"size": 1112094,
"sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1",
"executable": false
},
{
@@ -71,8 +71,8 @@
"muos": {
"archive": {
"dest": "ARCHIVE/RetroBIOS.muxapp",
"size": 1156738,
"sha256": "35c0920008baadca997d0f778e99ce46d702bbe3d3f0d33104d7158422b292aa"
"size": 1165721,
"sha256": "1b6ea28a5fc2f413aa7444ddb9ebe00b4b89c29ef639d62e3e1cf4d10793ed16"
},
"members": [
{
@@ -92,8 +92,8 @@
{
"name": "RetroBIOS/install.py",
"repo_path": "install.py",
"size": 1103111,
"sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858",
"size": 1112094,
"sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1",
"executable": false
},
{
@@ -166,8 +166,8 @@
{
"dest": ".config/modules/retrobios/install.py",
"repo_path": "install.py",
"size": 1103111,
"sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858",
"size": 1112094,
"sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1",
"executable": false
},
{
install/myminui.json
+9 -5
View File
@@ -6,7 +6,7 @@
"platform": "myminui",
"display_name": "MyMinUI",
"version": "2026-08-09",
"generated": "2026-11-05T18:33:47Z",
"generated": "2026-11-05T23:59:37Z",
"base_destination": "",
"detect": [
{
@@ -18957,7 +18957,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-cheats",
@@ -32686,7 +32687,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -32919,7 +32921,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "fbneo-samples",
@@ -33152,7 +33155,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
}
]
}
install/nextui.json
+7 -4
View File
@@ -6,7 +6,7 @@
"platform": "nextui",
"display_name": "NextUI",
"version": "2026-10-20",
"generated": "2026-11-05T18:33:47Z",
"generated": "2026-11-05T23:59:37Z",
"base_destination": "",
"detect": [
{
@@ -5635,7 +5635,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "fbneo-cheats",
@@ -19364,7 +19365,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -19597,7 +19599,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
}
]
}
install/recalbox.json
+4 -2
View File
@@ -62833,7 +62833,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -63066,7 +63067,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
}
],
"save_directory": {
install/retroarch.json
+32 -10
View File
@@ -21,12 +21,24 @@
"config": "$HOME/snap/retroarch/current/.config/retroarch/retroarch.cfg",
"parse_key": "system_directory"
},
{
"os": "linux",
"method": "config_file",
"config": "$XDG_CONFIG_HOME/retroarch/retroarch.cfg",
"parse_key": "system_directory"
},
{
"os": "linux",
"method": "config_file",
"config": "$HOME/.config/retroarch/retroarch.cfg",
"parse_key": "system_directory"
},
{
"os": "linux",
"method": "config_file",
"config": "$HOME/.retroarch.cfg",
"parse_key": "system_directory"
},
{
"os": "darwin",
"method": "config_file",
@@ -116911,7 +116923,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "cemu-gameprofiles",
@@ -117867,7 +117880,8 @@
"size": 36
}
},
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default"
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default",
"archive_size": 21360579
},
{
"ref": "dinothawr",
@@ -118315,7 +118329,8 @@
"size": 105
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 5752250
},
{
"ref": "dirksimple",
@@ -118371,7 +118386,8 @@
"size": 12988
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 175384
},
{
"ref": "dolphin-sys",
@@ -127042,7 +127058,8 @@
"strip_components": 2,
"exclude": [
"Themes"
]
],
"archive_size": 3197914
},
{
"ref": "fbneo-cheats",
@@ -140771,7 +140788,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -141004,7 +141022,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -141376,7 +141395,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "ppsspp-assets",
@@ -142064,7 +142084,8 @@
"size": 15005
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 9844388
},
{
"ref": "yaps2-resources",
@@ -142524,7 +142545,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
],
"save_directory": {
install/retrobat.json
+6 -3
View File
@@ -89075,7 +89075,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -89308,7 +89309,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -89680,7 +89682,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
}
],
"save_directory": {
install/retrodeck.json
+10 -5
View File
@@ -137751,7 +137751,8 @@
"size": 36
}
},
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default"
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default",
"archive_size": 21360579
},
{
"ref": "fbneo-cheats",
@@ -151480,7 +151481,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -151713,7 +151715,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -152085,7 +152088,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "yaps2-resources",
@@ -152549,7 +152553,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
],
"save_directory": {
install/retropie.json
+12 -6
View File
@@ -124071,7 +124071,8 @@
"size": 36
}
},
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default"
"source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default",
"archive_size": 21360579
},
{
"ref": "dolphin-sys",
@@ -132742,7 +132743,8 @@
"strip_components": 2,
"exclude": [
"Themes"
]
],
"archive_size": 3197914
},
{
"ref": "fbneo-cheats",
@@ -146471,7 +146473,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -146704,7 +146707,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -147076,7 +147080,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "yaps2-resources",
@@ -147536,7 +147541,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
]
}
install/rocknix.json
+20 -10
View File
@@ -86538,7 +86538,8 @@
"size": 344
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 4465709
},
{
"ref": "dinothawr",
@@ -86986,7 +86987,8 @@
"size": 105
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 5752250
},
{
"ref": "dirksimple",
@@ -87042,7 +87044,8 @@
"size": 12988
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 175384
},
{
"ref": "dolphin-sys",
@@ -95719,7 +95722,8 @@
"strip_components": 2,
"exclude": [
"Themes"
]
],
"archive_size": 3197914
},
{
"ref": "fbneo-cheats",
@@ -109448,7 +109452,8 @@
"size": 10054
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 5083767
},
{
"ref": "fbneo-samples",
@@ -109681,7 +109686,8 @@
"size": 551776
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 53006540
},
{
"ref": "freeintv-overlays",
@@ -110053,7 +110059,8 @@
"size": 248636
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 21311170
},
{
"ref": "lrps2",
@@ -110069,7 +110076,8 @@
"size": 1814032
}
},
"strip_components": 0
"strip_components": 0,
"archive_size": 318896
},
{
"ref": "ppsspp-assets",
@@ -110757,7 +110765,8 @@
"size": 15005
}
},
"strip_components": 1
"strip_components": 1,
"archive_size": 9844388
},
{
"ref": "yaps2-resources",
@@ -111217,7 +111226,8 @@
"size": 637
}
},
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources"
"source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources",
"archive_size": 21181421
}
]
}
platforms/_registry.yml
+13
View File
@@ -219,10 +219,23 @@ platforms:
method: config_file
config: $HOME/snap/retroarch/current/.config/retroarch/retroarch.cfg
parse_key: system_directory
# Native: $XDG_CONFIG_HOME/retroarch when the variable is set, else
# $HOME/.config/retroarch, then $HOME/.retroarch.cfg when that file is
# missing (file_path_special.c fill_pathname_application_data,
# configuration.c open_default_config_file). A value left at default
# is the system folder of the first of the two folders.
- os: linux
method: config_file
config: $XDG_CONFIG_HOME/retroarch/retroarch.cfg
parse_key: system_directory
- os: linux
method: config_file
config: $HOME/.config/retroarch/retroarch.cfg
parse_key: system_directory
- os: linux
method: config_file
config: $HOME/.retroarch.cfg
parse_key: system_directory
# configuration.c reads <Application Support>/RetroArch/config, and a value
# left at default is ~/Documents/RetroArch/system (platform_darwin.m).
# Older builds kept the file one level up.
schemas/install-manifest.schema.json
+5
View File
@@ -429,6 +429,11 @@
"minimum": 0,
"description": "With type zip: leading path parts dropped from each member."
},
"archive_size": {
"type": "integer",
"minimum": 1,
"description": "Bytes of the source archive when it was last downloaded: what fetching the tree costs. Absent when not recorded."
},
"source_path": {
"type": "string",
"description": "With type tarball: only members below this path are kept, relative to it."
scripts/generate_pack.py
+15 -1
View File
@@ -4949,9 +4949,12 @@ def _manifest_data_directories(
(strip_components for a ZIP, source_path for a tarball, exclude) come
from _data_dirs.yml, the same refresh_data_dirs applies. Every file
carries the sha1 and size the cache holds, so a member the source
changed since is refused rather than installed.
changed since is refused rather than installed. archive_size is the
size of the archive refresh_data_dirs last downloaded, what fetching
the tree costs: a tarball whose tree is 10 KB is a download of 21 MB.
"""
grouped: dict[tuple[str, str], dict] = {}
versions: dict[str, dict[str, dict]] = {}
for src, full in members:
ref, systems, prefix = refs[full]
entry = registry.get(ref, {})
@@ -4976,6 +4979,17 @@ def _manifest_data_directories(
group["source_path"] = str(entry.get("source_path", "")).format(version=version)
if entry.get("exclude"):
group["exclude"] = [str(e) for e in entry["exclude"]]
cache = entry.get("local_cache")
if cache:
recorded = Path(cache).parent / ".versions.json"
if str(recorded) not in versions:
try:
versions[str(recorded)] = json.loads(recorded.read_text())
except FileNotFoundError:
versions[str(recorded)] = {}
size = versions[str(recorded)].get(ref, {}).get("archive_size")
if isinstance(size, int) and size > 0:
group["archive_size"] = size
group["systems"].extend(s for s in systems if s not in group["systems"])
rel = full[len(prefix):].lstrip("/") if prefix else full
hashes = compute_hashes(src, frozenset({"sha1"}))
scripts/refresh_data_dirs.py
+62 -5
View File
@@ -24,7 +24,7 @@ import time
import urllib.error
import urllib.request
import zipfile
from collections.abc import Iterator
from collections.abc import Callable, Iterator
from pathlib import Path
from artifacts import file_lock as _file_lock, set_plain_mode
from common import InputMissing, github_token, platform_arg, require_platform, suggest_names, yaml_load
@@ -288,10 +288,12 @@ def _download_and_extract(
source_path: str,
local_cache: str,
exclude: list[str] | None = None,
measured: Callable[[int], None] | None = None,
) -> int:
"""Download tarball, extract source_path subtree to local_cache.
Returns the number of files extracted.
Returns the number of files extracted. measured receives the size of
the archive downloaded.
"""
exclude = exclude or []
cache_dir = Path(local_cache)
@@ -308,6 +310,8 @@ def _download_and_extract(
if not chunk:
break
f.write(chunk)
if measured is not None:
measured(tarball_path.stat().st_size)
log.info("extracting %s -> %s", source_path, local_cache)
@@ -365,6 +369,7 @@ def _download_and_extract_zip(
local_cache: str,
exclude: list[str] | None = None,
strip_components: int = 0,
measured: Callable[[int], None] | None = None,
) -> int:
"""Download ZIP, extract to local_cache. Returns file count.
@@ -387,6 +392,8 @@ def _download_and_extract_zip(
if not chunk:
break
f.write(chunk)
if measured is not None:
measured(zip_path.stat().st_size)
extract_dir = Path(tmpdir) / "extract"
extract_dir.mkdir()
@@ -426,6 +433,46 @@ def _download_and_extract_zip(
return file_count
def _measure_archive(source_url: str) -> int | None:
"""The size of the archive at source_url, read by streaming it: a
tarball GitHub builds on request is sent without a Content-Length."""
req = urllib.request.Request(source_url, headers={"User-Agent": USER_AGENT})
size = 0
try:
with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp:
while chunk := resp.read(65536):
size += len(chunk)
except _TRANSIENT as exc:
log.warning("could not measure %s: %s", source_url, exc)
return None
return size or None
def _record_archive_size(key: str, entry: dict, versions_path: str) -> None:
"""Record the archive size of a tree current since before it was kept.
The installer announces a tree's download as that size: a tarball
whose tree is 10 KB is a download of 21 MB. Measured, not refetched,
so the tree the packs carry stays the one recorded.
"""
path = Path(versions_path)
with _file_lock(path.with_name(f".{path.name}.lock")):
versions = _load_versions(versions_path)
record = versions.get(key)
if record is None or "archive_size" in record:
return
version = entry.get("version", "master")
size = _measure_archive(entry["source_url"].format(version=version))
if size is None:
return
with _file_lock(path.with_name(f".{path.name}.lock")):
versions = _load_versions(versions_path)
if key in versions and "archive_size" not in versions[key]:
versions[key]["archive_size"] = size
_save_versions(versions, versions_path)
log.info("[%s] archive size recorded: %d bytes", key, size)
def _get_remote_etag(source_url: str) -> str | None:
"""HEAD request to get ETag or Last-Modified for freshness check."""
try:
@@ -474,6 +521,7 @@ def refresh_entry(
return None
if not needs:
log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag))
_record_archive_size(key, entry, versions_path)
return False
with cache_lock(entry["local_cache"]):
return _refresh_entry(key, entry, force, dry_run, versions_path)
@@ -534,13 +582,19 @@ def _refresh_entry(
else:
remote_tag = get_remote_sha(entry["source_url"], version)
archive_sizes: list[int] = []
def fetch() -> int:
archive_sizes.clear()
if source_type == "zip":
return _download_and_extract_zip(
source_url, local_cache, exclude, entry.get("strip_components", 0)
source_url, local_cache, exclude, entry.get("strip_components", 0),
measured=archive_sizes.append,
)
source_path = entry["source_path"].format(version=version)
return _download_and_extract(source_url, source_path, local_cache, exclude)
return _download_and_extract(
source_url, source_path, local_cache, exclude, measured=archive_sizes.append
)
# An archive that reads but holds nothing under the path is the same on
# every try: only a failed or unreadable download is tried again.
@@ -558,7 +612,10 @@ def _refresh_entry(
log.warning("[%s] download failed (%s), trying again in %d s", key, exc, wait)
time.sleep(wait)
_record_version(key, {"sha": remote_tag or "", "version": version}, versions_path)
record = {"sha": remote_tag or "", "version": version}
if archive_sizes:
record["archive_size"] = archive_sizes[-1]
_record_version(key, record, versions_path)
log.info("[%s] refreshed: %d files extracted to %s", key, file_count, local_cache)
return True
tests/test_install.py
+277 -6
View File
@@ -451,6 +451,64 @@ class AMissingDestIsSaid(unittest.TestCase):
self.assertIn("does not exist", out)
class ADestThatIsAFileIsNamed(unittest.TestCase):
"""--dest naming a file, or a path under one, failed at mkdir with
EEXIST or ENOTDIR, and the run said this user cannot write there and
offered a sudo command that fails the same way."""
def _run(self, dest: Path) -> subprocess.CompletedProcess:
env = dict(os.environ, RETROBIOS_OS="linux")
return subprocess.run(
[sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch",
"--dest", str(dest), "--manifest", str(REPO_ROOT / "install" / "retroarch.json"),
"--system", "nintendo-gba", "--yes", "--events", "--offline"],
env=env, capture_output=True, text=True, timeout=120, check=False,
stdin=subprocess.DEVNULL)
def test_the_file_is_named_and_sudo_is_not_offered(self):
root = Path(_tempdir(self))
(root / "scph5501.bin").write_bytes(b"x")
for dest, words in (
(root / "scph5501.bin", f"{root / 'scph5501.bin'} is a file, not a folder"),
(root / "scph5501.bin" / "sub",
f"cannot be a folder: {root / 'scph5501.bin'} is a file"),
):
with self.subTest(dest=dest):
proc = self._run(dest)
self.assertEqual(proc.returncode, 1, proc.stderr)
self.assertIn(words, proc.stderr)
self.assertNotIn("sudo", proc.stderr)
errors = [json.loads(line) for line in proc.stdout.splitlines()
if line.startswith("{") and '"error"' in line]
self.assertEqual([(e["stage"], e["reason"]) for e in errors],
[("dest", "not_a_folder")])
self.assertEqual((root / "scph5501.bin").read_bytes(), b"x")
class SaveFilesAListHoldsNoneOfAreNotPromised(unittest.TestCase):
"""RetroPie, ROCKNIX, AmberELEC, EmuELEC and RomM keep their save
directory in no fixed place: their lists leave those files out with no
hash or address. The run said "Name it with --save-dir", and said it
again with --save-dir given, which installs none of them."""
def test_the_reason_is_said_and_no_option_is_offered(self):
manifest = json.loads((REPO_ROOT / "install" / "retropie.json").read_text())
self.assertIsNone(manifest.get("save_directory"))
root = Path(_tempdir(self))
env = dict(os.environ, RETROBIOS_OS="linux")
for extra in ([], ["--save-dir", str(root / "saves")]):
with self.subTest(extra=extra):
proc = _install_run(
env, "--platform", "retropie", "--dest", str(root / "bios"),
"--manifest", "install/retropie.json", "--offline", "--no-input",
"--yes", "--emulator", "azahar", *extra)
out = proc.stdout + proc.stderr
self.assertIn("read from RetroPie's save directory left out: RetroPie "
"keeps it in no fixed place, and its file list holds none "
"of them.", out)
self.assertNotIn("Name it with --save-dir", out)
class AFolderThatRefusesWritesIsNamed(unittest.TestCase):
"""A read-only BIOS folder ended the run on a traceback (a subfolder to
create), or on three retries per file and "once that is resolved", and a
@@ -2673,6 +2731,52 @@ class RetroArchWhereItReallyLives(unittest.TestCase):
self.assertIn(("retroarch", support / "system"), found)
class LinuxRetroArchFollowsXdgConfigHome(unittest.TestCase):
"""RetroArch on Linux keeps its config under $XDG_CONFIG_HOME/retroarch
when the variable is set, then falls back to ~/.retroarch.cfg
(file_path_special.c fill_pathname_application_data, configuration.c
open_default_config_file). Only ~/.config/retroarch was read: a home
with its own XDG layout detected nothing."""
def _env(self) -> tuple[Path, Path]:
home = Path(_tempdir(self))
xdg = Path(_tempdir(self))
patcher = unittest.mock.patch.dict(os.environ, {"HOME": str(home)}, clear=True)
patcher.start()
self.addCleanup(patcher.stop)
return home, xdg
def test_the_config_under_xdg_config_home_is_read(self):
home, xdg = self._env()
(xdg / "retroarch").mkdir()
(xdg / "retroarch" / "retroarch.cfg").write_text('system_directory = "/srv/bios"\n')
# A config under ~/.config is not the one this RetroArch opens.
(home / ".config" / "retroarch").mkdir(parents=True)
(home / ".config" / "retroarch" / "retroarch.cfg").write_text(
'system_directory = "/elsewhere"\n')
os.environ["XDG_CONFIG_HOME"] = str(xdg)
found = install.detect_platforms("linux")
self.assertIn(("retroarch", Path("/srv/bios")), found)
self.assertNotIn(("retroarch", Path("/elsewhere")), found)
self.assertEqual(install._os_default_dests("linux")["retroarch"],
xdg / "retroarch" / "system")
def test_home_retroarch_cfg_is_the_fallback(self):
home, xdg = self._env()
os.environ["XDG_CONFIG_HOME"] = str(xdg)
(home / ".retroarch.cfg").write_text('system_directory = "default"\n')
found = install.detect_platforms("linux")
self.assertIn(("retroarch", xdg / "retroarch" / "system"), found)
def test_without_the_variable_the_folder_is_under_dot_config(self):
home, _ = self._env()
(home / ".config" / "retroarch").mkdir(parents=True)
(home / ".config" / "retroarch" / "retroarch.cfg").write_text(
'system_directory = "default"\n')
found = install.detect_platforms("linux")
self.assertIn(("retroarch", home / ".config" / "retroarch" / "system"), found)
class NoPythonNamesThisSystemsCommand(unittest.TestCase):
"""Without Python the bootstrap said `sudo apt install python3` on every
system: Termux has no sudo and installs with `pkg install python`, and
@@ -3790,6 +3894,96 @@ class DataDirectoriesAreInstalled(unittest.TestCase):
(len(body), len(body)))
emit.assert_not_called()
def test_a_cut_archive_resumes_where_it_stopped(self):
"""A data archive cut at 90% was deleted and fetched again from
zero, on the retry and on the next run, while the run said that
nothing was lost."""
work = Path(_tempdir(self))
body = self._zip(work).read_bytes()
keep = work / ".dolphin-sys.archive.part"
requests: list[dict] = []
class Response(io.BytesIO):
def __init__(self, data, status, headers):
super().__init__(data)
self.status, self.headers = status, headers
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def urlopen(request, timeout=None):
sent = dict(request.header_items())
requests.append(sent)
if "Range" in sent:
start = int(sent["Range"][len("bytes="):-1])
if sent.get("If-range") != '"v1"':
return Response(body, 200, {"ETag": '"v2"'})
return Response(body[start:], 206, {
"Content-Range": f"bytes {start}-{len(body) - 1}/{len(body)}",
"Content-Length": str(len(body) - start), "ETag": '"v1"'})
cut = len(body) * 9 // 10
class Cut(Response):
def read(self, size=-1):
if self.tell() >= cut:
raise ConnectionResetError("cut")
return super().read(min(size, cut - self.tell()))
return Cut(body, 200, {"Content-Length": str(len(body)), "ETag": '"v1"'})
with unittest.mock.patch.object(install.urllib.request, "urlopen", urlopen), \
unittest.mock.patch.object(install, "_network_allowed", lambda url: None), \
unittest.mock.patch.object(install, "_retry_delay", lambda *a: 0):
# The next run, after a stop: what the cut attempt wrote is kept.
with unittest.mock.patch.object(install, "MAX_RETRIES", 1), \
self.assertRaises(ConnectionResetError):
install._fetch_archive("https://example.invalid/D.zip", work, None, keep)
self.assertEqual(keep.stat().st_size, len(body) * 9 // 10)
path = install._fetch_archive("https://example.invalid/D.zip", work, None, keep)
self.assertEqual(path.read_bytes(), body)
self.assertEqual(requests[-1]["Range"], f"bytes={len(body) * 9 // 10}-")
self.assertEqual(requests[-1]["If-range"], '"v1"')
def test_a_changed_archive_is_fetched_whole(self):
work = Path(_tempdir(self))
body = self._zip(work).read_bytes()
keep = work / ".dolphin-sys.archive.part"
keep.write_bytes(b"older revision")
keep.with_name(keep.name + ".source").write_text(
'https://example.invalid/D.zip\n"v0"\n')
class Whole(io.BytesIO):
status, headers = 200, {"ETag": '"v1"'}
def __enter__(self):
return self
def __exit__(self, *exc):
return False
with unittest.mock.patch.object(install.urllib.request, "urlopen",
lambda request, timeout=None: Whole(body)), \
unittest.mock.patch.object(install, "_network_allowed", lambda url: None):
path = install._fetch_archive("https://example.invalid/D.zip", work, None, keep)
self.assertEqual(path.read_bytes(), body)
def test_a_tree_read_whole_leaves_no_archive(self):
work = Path(_tempdir(self))
archive = self._zip(work)
bios = work / "system"
bios.mkdir()
def fetch(url, directory, progress=None, keep=None):
shutil.copy(archive, keep)
keep.with_name(keep.name + ".source").write_text(f"{url}\n\"v1\"\n")
return keep
with unittest.mock.patch.object(install, "_fetch_archive", fetch):
install.install_data_directory(self._entry(), bios)
self.assertEqual([p.name for p in (bios / "dolphin-emu").iterdir()], ["Sys"])
def test_the_run_ends_on_the_archive_bytes_too(self):
from test_install_selection import FakeRepository # noqa: PLC0415
@@ -3811,13 +4005,44 @@ class DataDirectoriesAreInstalled(unittest.TestCase):
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertIn(f"{install.format_size(bios + archive)} downloaded in", proc.stdout)
def test_the_plan_announces_the_archive_not_the_tree(self):
"""yaps2's tarball is 21.2 MB for a 6.5 MB tree, Cemu's 21.4 MB for
9.9 KB: the plan announced the tree, and the run downloaded far more."""
from test_install_selection import FakeRepository # noqa: PLC0415
member = b"profile"
archive_size = 21_360_579
with FakeRepository() as repo:
listed = json.loads((repo.root / "install" / "retroarch.json").read_text())
listed["data_directories"] = [{
"ref": "cemu-gameprofiles", "dest": "cemu", "type": "tarball",
"url": "https://example.invalid/Cemu.tar.gz", "source_path": "Cemu-1/bin",
"systems": ["nintendo-gba"], "archive_size": archive_size,
"files": {"gameProfiles/a.ini": {"sha1": hashlib.sha1(member).hexdigest(),
"size": len(member)}}}]
(repo.root / "install" / "retroarch.json").write_text(json.dumps(listed))
proc = repo.run("--no-input", "--dry-run")
bios = sum(len(payload) for payload, *_ in FakeRepository.PAYLOADS.values())
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertIn(f"({install.format_size(bios + archive_size)})", proc.stdout)
self.assertIn(f"(a {install.format_size(archive_size)} archive)", proc.stdout)
def test_a_pack_or_an_old_list_counts_the_files_themselves(self):
needed = {"a": {"size": 7}}
entry = {"archive_size": 21_360_579}
self.assertEqual(install.data_fetch_bytes(entry, needed), 21_360_579)
self.assertEqual(install.data_fetch_bytes({}, needed), 7)
self.assertEqual(install.data_fetch_bytes(entry, {}), 0)
with unittest.mock.patch.object(install.LOCAL, "pack", object()):
self.assertEqual(install.data_fetch_bytes(entry, needed), 7)
def test_verified_members_land_the_rest_are_said(self):
work = Path(_tempdir(self))
archive = self._zip(work)
bios = work / "system"
bios.mkdir()
def fetch(url, directory, progress=None):
def fetch(url, directory, progress=None, keep=None):
copy = Path(directory) / "a.zip"
shutil.copy(archive, copy)
return copy
@@ -3853,8 +4078,9 @@ class DataDirectoriesAreInstalled(unittest.TestCase):
def members(_archive, _entry):
yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Endless())
with unittest.mock.patch.object(install, "_fetch_archive",
lambda url, d, progress=None: Path(d) / "a"), \
with unittest.mock.patch.object(
install, "_fetch_archive",
lambda url, d, progress=None, keep=None: Path(d) / "a"), \
unittest.mock.patch.object(install, "_archive_members", members):
counts = install.install_data_directory(entry, work)
self.assertEqual(counts["changed"], 1)
@@ -3881,8 +4107,9 @@ class DataDirectoriesAreInstalled(unittest.TestCase):
def members(_archive, _entry):
yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Broken())
with unittest.mock.patch.object(install, "_fetch_archive",
lambda url, d, progress=None: Path(d) / "a"), \
with unittest.mock.patch.object(
install, "_fetch_archive",
lambda url, d, progress=None, keep=None: Path(d) / "a"), \
unittest.mock.patch.object(install, "_archive_members", members):
counts = install.install_data_directory(entry, work)
self.assertTrue(counts["error"])
@@ -4422,6 +4649,50 @@ def _install_run(env: dict, *argv: str, **kwargs) -> subprocess.CompletedProcess
)
class AListingAppliesItsOptionsOrRefusesThem(unittest.TestCase):
"""--list-systems --list-cores printed the systems alone, and
--list-platforms, --list-targets and --list-emulators printed the same
list with --system, --region or --core as without, code 0."""
def test_two_listings_or_an_option_a_listing_ignores_are_refused(self):
env = dict(os.environ, RETROBIOS_OS="linux")
given = ["--platform", "retroarch", "--offline", "--no-input",
"--manifest", "install/retroarch.json"]
for argv, words in (
([*given, "--list-systems", "--list-cores"],
"--list-systems and --list-cores each print a list"),
([*given, "--list-cores", "--list-emulators"],
"--list-cores and --list-emulators each print a list"),
(["--list-platforms", "--list-systems", "--platform", "retroarch"],
"--list-platforms and --list-systems each print a list"),
(["--list-platforms", "--region", "us"],
"--list-platforms prints the whole list, which --region does not narrow"),
(["--list-platforms", "--platform", "retroarch"],
"--list-platforms prints the whole list, which --platform does not narrow"),
([*given, "--list-targets", "--system", "sony-playstation"],
"--list-targets prints the whole list, which --system does not narrow"),
([*given, "--list-targets", "--target", "switch"],
"--list-targets prints the whole list, which --target does not narrow"),
([*given, "--list-emulators", "--system", "sony-playstation"],
"--list-emulators prints the whole list, which --system does not narrow"),
([*given, "--list-emulators", "--core", "beetle_psx"],
"--list-emulators prints the whole list, which --core does not narrow"),
):
with self.subTest(argv=argv):
proc = _install_run(env, *argv)
self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr)
self.assertIn(words, proc.stderr)
def test_the_options_a_listing_reads_still_apply(self):
env = dict(os.environ, RETROBIOS_OS="linux")
given = ["--platform", "retroarch", "--offline", "--no-input",
"--manifest", "install/retroarch.json"]
whole = _install_run(env, *given, "--list-cores")
narrowed = _install_run(env, *given, "--list-cores", "--system", "nintendo-gba")
self.assertEqual((whole.returncode, narrowed.returncode), (0, 0), narrowed.stderr)
self.assertLess(len(narrowed.stdout), len(whole.stdout))
class AMissingPackNamesTheOptionAndThePacksBeside(unittest.TestCase):
"""A typo in --from-pack said "Error: X does not exist": not the option,
not the pack sitting next to the name typed."""
@@ -6562,7 +6833,7 @@ class AFileYouHadIsKept(unittest.TestCase):
"type": "zip", "files": {"Sys/a.ini": {
"sha1": hashlib.sha1(self.NEW).hexdigest(), "size": len(self.NEW)}}}
def fetch(url, directory, progress=None):
def fetch(url, directory, progress=None, keep=None):
copy = Path(directory) / "a.zip"
shutil.copy(archive, copy)
return copy
tests/test_install_offline.py
+6 -1
View File
@@ -778,7 +778,12 @@ class OfflineMakesNoRequest(Scratch):
self.assertIn("--offline", proc.stderr)
def test_a_listing_and_a_pack_need_no_network_anyway(self):
proc = self.run_installer("--offline", "--list-platforms")
# --platform alone with it is refused: the listing names every one.
proc = subprocess.run(
[sys.executable, str(REPO_ROOT / "install.py"), "--offline", "--list-platforms"],
env=_offline_env(), capture_output=True, text=True, timeout=120,
check=False, stdin=subprocess.DEVNULL,
)
self.assertEqual(proc.returncode, 0, proc.stderr)
tests/test_install_tui.py
+55 -9
View File
@@ -2967,12 +2967,44 @@ class SaveDirectoryLeftOut(unittest.TestCase):
UNPLACED = {"reason": "no_save_directory", "label": "no save directory", "files": 2,
"required": 0}
def _ended(self, manifest: dict) -> tuple:
state = fresh_state()
state.stack = ["setup"]
with mock.patch.object(install.Path, "exists", return_value=True):
install.tui_update(state, Key("enter"))
install.tui_receive(state, ("manifest", "retroarch", manifest))
summary = {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 0,
"pending": 5, "failed": [], "skipped": [self.UNPLACED], "left_out": [],
"kept": [], "interrupted": False}
for event in [*FILES, *SYSTEMS, CHECK, summary]:
install.tui_receive(state, ("event", state.run_id, event))
with mock.patch.object(install, "_tui_folder_facts", return_value={
"free": 50 * 1024 ** 3, "writable": True, "cores": []}):
install.tui_receive(state, ("ended", state.run_id, 1, []))
return state, summary
def test_a_list_that_holds_none_of_them_offers_no_folder(self):
"""RetroPie's list leaves its save files out with no hash or address:
--save-dir installs none of them, and was advised again each run."""
state, summary = self._ended(MANIFEST)
line = ("2 files read from RetroArch's save directory left out: RetroArch keeps "
"it in no fixed place, and its file list holds none of them.")
plan = " ".join(" ".join(install.tui_view(state, 120, 40)).split())
self.assertIn(line, plan)
self.assertNotIn("--save-dir", plan)
state.stack.append("result")
state.code = 0
state.summary = {**summary, "mode": "install", "installed": 5}
self.assertEqual([text for text in install.tui_outcome(state)[1] if "save" in text],
[line])
def test_files_left_out_of_the_save_directory_are_said(self):
state = fresh_state()
state.stack = ["setup"]
with mock.patch.object(install.Path, "exists", return_value=True):
install.tui_update(state, Key("enter"))
install.tui_receive(state, ("manifest", "retroarch", MANIFEST))
install.tui_receive(state, ("manifest", "retroarch", {
**MANIFEST, "save_directory": {"folder": "saves", "by_core": True}}))
summary = {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 0,
"pending": 5, "failed": [], "skipped": [self.UNPLACED], "left_out": [],
"kept": [], "interrupted": False}
@@ -3834,6 +3866,8 @@ class BackTicksAndSaveFolder(unittest.TestCase):
def test_the_save_folder_remedy_is_the_same_on_both_screens(self):
summary = {"skipped": [{"reason": "no_save_directory", "files": 1}]}
state = checked_state()
state.manifest = {**state.manifest, "save_directory": {"folder": "saves",
"by_core": True}}
lines = install._unplaced_lines(summary, state)
self.assertIn("Options, Save folder (--save-dir)", lines[0])
@@ -5898,7 +5932,10 @@ def _card_from_rules(root: Path, setup: install.CardSetup) -> Path:
(root / "bios").mkdir(exist_ok=True)
for value in rule.values:
(root / value).parent.mkdir(parents=True, exist_ok=True)
if rule.method == "games_part" and "." not in PurePosixPath(value).name:
# A marker the BIOS folder sits under is a folder on the card
# (muOS's MUOS/bios): written as a file, no folder fits below it.
if (rule.method == "games_part" and "." not in PurePosixPath(value).name
or str(setup.below).startswith(f"{value}/")):
(root / value).mkdir(exist_ok=True)
else:
(root / value).write_text("", encoding="utf-8")
@@ -9763,14 +9800,21 @@ class ScreenRowsAndFilters(unittest.TestCase):
from test_install_offline import PAYLOAD, _offline_env, _pack
blocker = self.root / "file"
blocker.write_text("not a folder")
if os.name == "nt" or os.geteuid() == 0:
self.skipTest("a folder that refuses writes is POSIX, and root writes anyway")
# A file in the way is refused before the plan, as not_a_folder.
blocker = self.root / "closed"
blocker.mkdir()
pack = _pack(self.root, "RetroArch_v1_BIOS_Pack.zip", PAYLOAD)
proc = subprocess.run(
[sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch",
"--dest", str(blocker / "system"), "--from-pack", str(pack), "--events"],
env=_offline_env(), capture_output=True, text=True, timeout=120, check=False,
stdin=subprocess.DEVNULL)
os.chmod(blocker, 0o500)
try:
proc = subprocess.run(
[sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch",
"--dest", str(blocker / "system"), "--from-pack", str(pack), "--events"],
env=_offline_env(), capture_output=True, text=True, timeout=120,
check=False, stdin=subprocess.DEVNULL)
finally:
os.chmod(blocker, 0o700)
events = [json.loads(line) for line in proc.stdout.splitlines()]
kinds = [event["event"] for event in events]
self.assertEqual(kinds[-3:], ["error", "summary", "end"], proc.stderr)
@@ -11836,6 +11880,8 @@ class OneChoicePerRun(unittest.TestCase):
def test_the_farewell_names_the_option_not_the_screen(self):
state = _installed_state(SYSTEM_DIR)
state.manifest = {**state.manifest, "save_directory": {"folder": "saves",
"by_core": True}}
state.summary = {**state.summary, "skipped": [
{"reason": "no_save_directory", "label": "", "files": 1, "required": False}]}
farewell = " ".join(install.tui_farewell(state))
tests/test_refresh_data_dirs.py
+47 -2
View File
@@ -93,6 +93,26 @@ class RefreshConcurrency(unittest.TestCase):
)
self.assertEqual(seen, [True])
def test_the_archive_size_is_recorded_without_replacing_the_tree(self):
"""The installer announces a tree's download as the archive's size:
a tarball extracted to 10 KB is a download of 21 MB. A tree current
before the size was kept is measured, its files left as they are."""
payload = _zip_bytes({"a.bin": b"x" * 100})
entry = {"source_type": "zip", "source_url": "https://x/pak.zip",
"local_cache": str(self.cache)}
with mock.patch.object(rdd.urllib.request, "urlopen",
lambda *a, **k: _Response(payload)), \
mock.patch.object(rdd, "_get_remote_etag", return_value="e1"):
self.assertTrue(rdd.refresh_entry("pak", entry, versions_path=self.versions))
self.assertEqual(rdd._load_versions(self.versions)["pak"]["archive_size"],
len(payload))
rdd._record_version("pak", {"sha": "e1"}, self.versions)
(self.cache / "a.bin").write_bytes(b"kept")
self.assertFalse(rdd.refresh_entry("pak", entry, versions_path=self.versions))
self.assertEqual(rdd._load_versions(self.versions)["pak"],
{"sha": "e1", "archive_size": len(payload)})
self.assertEqual((self.cache / "a.bin").read_bytes(), b"kept")
def test_a_current_tree_is_not_waited_for(self):
"""A pipeline reads every tree for its whole run. A build beside it
waited on the exclusive lock before asking the remote, which would
@@ -100,7 +120,7 @@ class RefreshConcurrency(unittest.TestCase):
import fcntl # noqa: PLC0415
self.cache.mkdir(parents=True)
rdd._record_version("pak", {"sha": "e1"}, self.versions)
rdd._record_version("pak", {"sha": "e1", "archive_size": 1}, self.versions)
entry = {"source_type": "zip", "source_url": "https://x/pak.zip",
"local_cache": str(self.cache)}
result: list = []
@@ -258,7 +278,7 @@ class DryRunReport(unittest.TestCase):
def test_the_remote_tag_is_shown_beside_the_cached_one(self):
with tempfile.TemporaryDirectory() as tmp:
versions = Path(tmp) / "versions.json"
versions.write_text(json.dumps({"k": {"etag": '"2725165595"'}}))
versions.write_text(json.dumps({"k": {"etag": '"2725165595"', "archive_size": 1}}))
entry = {"source_type": "zip", "source_url": "https://x.invalid/a.zip",
"local_cache": tmp}
with mock.patch.object(rdd, "_get_remote_etag", return_value='"2685251384"'), \
@@ -278,5 +298,30 @@ class DryRunReport(unittest.TestCase):
)
class TheManifestCarriesTheArchiveSize(unittest.TestCase):
"""The size refresh_data_dirs recorded reaches the install list, which
the installer announces as the tree's download."""
def test_recorded_size_is_written_and_an_unrecorded_one_left_out(self):
from generate_pack import _manifest_data_directories # noqa: PLC0415
(REPO_ROOT / "tmp").mkdir(exist_ok=True)
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
data = Path(tmp) / "data"
for ref in ("pak", "old"):
(data / ref).mkdir(parents=True)
(data / ref / "a.bin").write_bytes(b"abc")
(data / ".versions.json").write_text(json.dumps(
{"pak": {"sha": "e1", "archive_size": 21_360_579}, "old": {"sha": "e2"}}))
registry = {ref: {"source_type": "zip", "source_url": f"https://x/{ref}.zip",
"local_cache": str(data / ref)} for ref in ("pak", "old")}
members = [(str(data / ref / "a.bin"), f"{ref}/a.bin") for ref in ("pak", "old")]
refs = {f"{ref}/a.bin": (ref, ("s",), ref) for ref in ("pak", "old")}
listed = {entry["ref"]: entry for entry in
_manifest_data_directories(members, refs, registry, "")}
self.assertEqual(listed["pak"]["archive_size"], 21_360_579)
self.assertNotIn("archive_size", listed["old"])
if __name__ == "__main__":
unittest.main()
wiki/installer.md
+21 -13
View File
@@ -352,8 +352,10 @@ file list is downloaded once and handed to each run.
times, after 1, 2 then 4 seconds or the delay a `Retry-After` header asks.
A file of 16 MB or more keeps its partial copy (`.<name>.part`) when a
transfer stops, and the next attempt or the next run requests only the
rest. A file already there with other contents is moved aside first, see
[Files you already had](#files-you-already-had).
rest. A data tree's archive does the same beside the tree
(`.<ref>.archive.part`), asking with `If-Range` so that a server whose
archive changed sends it whole. A file already there with other contents
is moved aside first, see [Files you already had](#files-you-already-had).
8. Copies into standalone emulator directories, only with `--standalone-copies`.
The file list names the version of the platform it was built for (Batocera
@@ -386,7 +388,10 @@ the save directory beside it (`saves`, `/storage/savefiles` on Lakka).
applies, or when `savefiles_in_content_dir` or
`sort_savefiles_by_content_enable` keeps saves beside each game, those files
are left out and the run says why, like the other entries the list cannot
install. The full screen says so on its plan and its result.
install. The full screen says so on its plan and its result. RetroPie, ROCKNIX,
AmberELEC, EmuELEC and RomM keep the save directory in no fixed place: their
lists hold none of these files, so `--save-dir` installs none, and the run says
that instead of naming the option.
Batocera, KNULLI, Recalbox, RetroBat and RetroDECK write the save directory at
each launch, one folder per system (`by_system` in the file list). Their lists
@@ -976,9 +981,9 @@ what the options name, everything by default.
| <span id="option-required-only"></span>`--required-only` | With `--for-games`, only the files the games cannot start without |
| <span id="option-no-game-data"></span>`--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) |
| <span id="option-declared-only"></span>`--declared-only` | Keep only the files the platform's own list names (`cores: null` in the manifest), not the extra files cores load. With `--check`, the answer is about the list the platform itself checks. Refused with `--core`, the listings it does not change, `--standalone-copies` and `--tui` |
| <span id="option-list-systems"></span>`--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart |
| <span id="option-list-cores"></span>`--list-cores` | Print the cores whose extra files a platform installs |
| <span id="option-list-emulators"></span>`--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for |
| <span id="option-list-systems"></span>`--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart. `--system`, `--core`, `--region`, `--target` and `--installed-cores` narrow it, as they narrow an install. A run prints one list: two listing options are refused |
| <span id="option-list-cores"></span>`--list-cores` | Print the cores whose extra files a platform installs. The options that narrow `--list-systems` narrow it too |
| <span id="option-list-emulators"></span>`--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for. `--target` and `--installed-cores` narrow it, and `--system`, `--core`, `--region` and `--emulator` are refused |
| <span id="option-romm-firmware"></span>`--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) |
| <span id="option-no-input"></span>`--no-input` | Never ask a question, even at a terminal. A run whose output goes to a file or a pipe asks nothing either, and says so on stderr. It finds the setups and the mounted cards the screen finds: one is taken, as Enter takes it, two stop the run with an `error` event (`stage` `detect`, `reason` `conflict`) naming each with its `--platform` and `--dest`, and none stops it too, with `reason` `not_found` when a file list or a pack names the platform |
| <span id="option-check"></span>`--check` | Report and exit without writing |
@@ -994,8 +999,8 @@ what the options name, everything by default.
| <span id="option-tui"></span>`--tui` | Open the full-screen interface, or stop with the reason it cannot open here (code 2) instead of falling back to the questions. Wins over `RETROBIOS_PLAIN` and `ACCESSIBLE`. See [The full-screen interface](#the-full-screen-interface) |
| <span id="option-plain"></span>`--plain` | Ask the questions line by line instead of the full-screen interface, the mode for a screen reader. See [The full-screen interface](#the-full-screen-interface) |
| <span id="option-version"></span>`--version` | Print the installer's identity, the first 12 hex digits of the SHA-256 the bootstraps pin |
| <span id="option-list-platforms"></span>`--list-platforms` | Print the supported platforms and what was detected here |
| <span id="option-list-targets"></span>`--list-targets` | Print the hardware targets a platform publishes, with the core count each carries |
| <span id="option-list-platforms"></span>`--list-platforms` | Print the supported platforms and what was detected here. An option that narrows a list (`--platform`, `--system`, `--core`, `--region`, `--emulator`, `--target`, `--every-board`, `--installed-cores`) is refused |
| <span id="option-list-targets"></span>`--list-targets` | Print the hardware targets a platform publishes, with the core count each carries. `--system`, `--core`, `--region`, `--emulator`, `--target`, `--every-board` and `--installed-cores` are refused |
| <span id="option-jobs"></span>`--jobs N`, `-j N` | Parallel downloads, 1 to 32, default 8 |
| <span id="option-verbose"></span>`--verbose`, `-v` | Print per-attempt failures and hash mismatches |
| <span id="option-standalone-copies"></span>`--standalone-copies` | Also copy into detected standalone emulator directories |
@@ -1617,8 +1622,11 @@ built (`strip_components` for a ZIP, `source_path` for a tarball, `exclude`),
and writes a file only when its hash matches the list. A file the source has
changed since is left out and counted, and the pack still holds it. `--check`
counts the data files missing, `--dry-run` names each tree it would fetch, and
narrowing with `--system`, `--core` or `--target` keeps only the trees of the
systems that stay.
the size announced for a tree is its archive's (`archive_size`, recorded when
`refresh_data_dirs.py` last downloaded it): one missing file fetches the whole
archive, and a tarball can be a thousand times its tree. Narrowing with
`--system`, `--core` or `--target` keeps only the trees of the systems that
stay.
## What the bootstrap does
@@ -1740,7 +1748,7 @@ fields, the installer refuses to narrow rather than install everything.
| Code | Meaning |
|------|---------|
| `0` | everything asked for is installed or already in place |
| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 with its ssl, unicodedata and zlib modules (nothing was run), an unusable base URL, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed or were left out by `--offline` while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, a `--dest` that is not the card of the system named (another system's card, a folder inside one, a drive with none of its files where its card always has them, or a folder that does not exist), `--restore` could not put a file back, or `--add-app` could not put every file of the app on the card |
| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 with its ssl, unicodedata and zlib modules (nothing was run), an unusable base URL, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed or were left out by `--offline` while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, a BIOS folder that is a file or lies under one, a `--dest` that is not the card of the system named (another system's card, a folder inside one, a drive with none of its files where its card always has them, or a folder that does not exist), `--restore` could not put a file back, or `--add-app` could not put every file of the app on the card |
| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), or `--offline` left every file out, the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board`, `--target` with `--from-pack` or `--offline` (or `RETROBIOS_INSTALL_OFFLINE` set) when no core list per board is at hand (the pack carries none and none is beside `--manifest` as `targets/<platform>.json`), or for a platform that publishes none, which is said to have no hardware targets, `--list-targets` with `--offline` and neither, `--add-app` with `--check`, `--dry-run`, a listing, `--from-pack` or `--offline`. The message then starts with `install.py: error:`. Also an unknown platform, target, system, core or emulator, said in an `Error:` or `Unknown platform` line and an `error` event whose `stage` is `option`. Also `--add-app` for a folder on no card that takes the app, said in an `Error:` line and an `app` event before anything is written. With `--restore`, a folder that is not a dated folder inside `retrobios-backup`. Also a quit chosen before the install, as `gh` ends a cancelled prompt: `q` at a question, `q` or **Quit** on the full screen, which then keeps `RetroBIOS: quit. Nothing was installed.`, and the full screen closed with no button pressed (`RETROBIOS_IDLE_EXIT`). No code but `0` closes a Windows Terminal tab started on the installer itself, so such a tab stays open on a quit |
| `130` | stopped with Ctrl+C, or with Ctrl+C or Ctrl+D at a question, the terminal keeping `RetroBIOS: stopped. Nothing was installed.` before an install. Run the same command again to carry on |
| `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) |
@@ -1787,7 +1795,7 @@ other schemas under `api/v1/schemas/`.
| `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` |
| `unsearched` | `folder`, `reason`, `message` | Before `detect`, once for each folder the search for setups could not look in, this user being refused it or a read failing there (a failing card, a link that loops). What it would hold counts as absent, and the other places are still searched: `detect` then lists what they hold, and an empty `platforms` after an `unsearched` line means nothing was found where the installer could look. `reason` says why in words (`Permission denied`), `message` says it in a sentence |
| `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found |
| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), `no_ssl` when this Python was built without its `ssl` module and cannot use HTTPS, or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). The BIOS folder sits under a folder this user cannot open (`stage` is `dest`, `reason` `unreadable`), and the run ends with code 1 before anything is written. A platform, target, system, core or emulator named on the command line is unknown or cannot be used here (`stage` is `option`, `reason` `unknown`, `platform` the platform it was checked against or the name typed), refused before any download with code 2. The BIOS folder cannot be created (`stage` is `dest`, `reason` `not_writable`): a `summary` follows with what the run installed before it. Two systems are detected on one machine and no `--dest` names the folder (`stage` is `detect`, `reason` `conflict`). The platform named with `--platform` is found nowhere and no folder can be assumed for it: a card system with no card named, a portable emulator, Batocera or Recalbox named on a computer (their folder is the `bios` folder of their SHARE drive), or no usual folder on this system or one set aside behind a folder this user cannot open (`stage` is `detect`, `reason` `not_found`). So is a platform named by a file list or a pack, in a run without questions where it is found nowhere here. A `--dest` refused as the card of the system named carries the same `stage` and `reason`: another system's card, a folder inside a card, a drive with none of the files its card always has, or a folder that does not exist. `message` says it in words, and is the only thing that tells these cases apart |
| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), `no_ssl` when this Python was built without its `ssl` module and cannot use HTTPS, or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). The BIOS folder sits under a folder this user cannot open (`stage` is `dest`, `reason` `unreadable`), or is a file or lies under one (`reason` `not_a_folder`), and the run ends with code 1 before anything is written. A platform, target, system, core or emulator named on the command line is unknown or cannot be used here (`stage` is `option`, `reason` `unknown`, `platform` the platform it was checked against or the name typed), refused before any download with code 2. The BIOS folder cannot be created (`stage` is `dest`, `reason` `not_writable`): a `summary` follows with what the run installed before it. Two systems are detected on one machine and no `--dest` names the folder (`stage` is `detect`, `reason` `conflict`). The platform named with `--platform` is found nowhere and no folder can be assumed for it: a card system with no card named, a portable emulator, Batocera or Recalbox named on a computer (their folder is the `bios` folder of their SHARE drive), or no usual folder on this system or one set aside behind a folder this user cannot open (`stage` is `detect`, `reason` `not_found`). So is a platform named by a file list or a pack, in a run without questions where it is found nowhere here. A `--dest` refused as the card of the system named carries the same `stage` and `reason`: another system's card, a folder inside a card, a drive with none of the files its card always has, or a folder that does not exist. `message` says it in words, and is the only thing that tells these cases apart |
| `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered |
| `version` | `platform`, `list_version`, `installed_version`, `message` | Before `choices`, when the system here runs another version than the one the list was built for: `list_version` is the list's, `installed_version` the one the system's version file states, and `message` says it and what it means. Not written when either version is unknown. The run goes on |
| `board` | `platform`, `board`, `applied`, `message` | Before `choices`, with no `--target` and no `--every-board`, when the system names its board or a Batocera or KNULLI card mounted here does (`boot/batocera.board` or `boot/knulli.board` on its boot partition, `board` then being the board its image maps to): `applied` is true when the run keeps that board's files only, false for a board the platform's list does not name, which keeps every board's files. `message` says it |
@@ -1801,7 +1809,7 @@ other schemas under `api/v1/schemas/`.
| `games_unavailable` | `platform`, `reason`, `message` | With `--for-games`, what the games need could not be read (`offline`, `network`, `invalid`, `stale`, `no_sets`, `no_folder`): the run keeps every file of the list, except for a `FOLDER` that does not exist, where it stops |
| `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply |
| `declared` | `platform`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `dests` | Right after `check`, the same counts over the files the platform's own list names (`cores: null`), and their `dests` in the order of the `file` events. A setup that checks its own list reads its answer here, with or without `--declared-only` |
| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes they write once in place |
| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes fetching them costs: each tree's archive, or the files themselves from a pack or a list that records no archive size |
| `found` | `platform`, `dest`, `folder`, `path`, `member`, `reason` | Each file to install that this machine already holds, after `check` and before anything is written, in `--check` and `--dry-run` too. `folder` is the `--from` folder as given, or the `bios_dir` of the check when the BIOS folder holds the file at another place, and then `reason` says where in words (`nested folder: ...` for a misplaced file, `the same file is at ...` for one the list installs there too). `path` is the file, and `member` the ZIP member holding the bytes when `path` is an archive, an empty string otherwise. Its copy is reported by a `downloaded` event |
| `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) |
| `saves` | `platform`, `files`, `folder` | After `choices` and before `checking`, when the files the run installs (once `--system` and the other choices narrowed the list) hold some read from the frontend's save directory and that directory is known: how many, and the `folder` they go to, where `<system>` and `<core>` stand for one folder per system or per core (the screen and `--plain` say it in words: "a folder per core under /sdcard/RetroArch/saves"). When it is not known, the files are counted in the summary's `skipped` instead (`no_save_directory`) |