mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
chore: merge the installer backups
This commit is contained in:
commit
5d868ee19b
13 files changed
+695
-41
No files matched your search
+1
-1
@@ -11,7 +11,7 @@
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
$defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
$defaultInstallSha256 = "6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a"
|
||||
$defaultInstallSha256 = "8bf9b5bf68dbde80025b1b4e1c18e31f43c016758df91be202613933f1de17d5"
|
||||
$maximumInstallerBytes = 2MB
|
||||
$installerExit = 0
|
||||
$installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }
|
||||
|
||||
+363
-14
@@ -485,9 +485,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = {
|
||||
# failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files",
|
||||
# "required"}], the files the collection cannot supply. left_out:
|
||||
# [{"category", "label", "files", "dests"}], what --no-game-data left
|
||||
# out, empty without it.
|
||||
# out, empty without it. kept: [{"folder", "files", "bytes"}], the files
|
||||
# in place the run moved aside before replacing them (under --dry-run,
|
||||
# would move), one per backup folder, empty when none.
|
||||
"summary": ("mode", "installed", "up_to_date", "pending", "failed", "skipped",
|
||||
"left_out", "interrupted"),
|
||||
"left_out", "kept", "interrupted"),
|
||||
"end": ("code",),
|
||||
}
|
||||
# Seconds between two progress events while bytes arrive.
|
||||
@@ -3429,6 +3431,279 @@ def check_local(
|
||||
return to_download, up_to_date, mismatched
|
||||
|
||||
|
||||
# Where a file in place that this run replaces is kept: a folder beside the
|
||||
# one written to, never inside it. Several emulators find their BIOS by
|
||||
# content, not by name: DuckStation, SwanStation and Rustation read every
|
||||
# file of the BIOS folder, BizHawk walks its firmware folder recursively. A
|
||||
# kept copy anywhere under that folder could be taken for the BIOS.
|
||||
BACKUP_DIR = "retrobios-backup"
|
||||
|
||||
|
||||
class _NotKept(Exception):
|
||||
"""The file in place differs and could not be set aside: it stays."""
|
||||
|
||||
|
||||
class KeptFiles:
|
||||
"""The files this run moved aside before replacing them.
|
||||
|
||||
A file is kept at <holder>/<run>/<folder name>/<path below the folder>,
|
||||
the holder being retrobios-backup beside the folder written to. The run
|
||||
folder is named by date and time, so a second run never mixes with the
|
||||
first, and copying its contents into the holder's parent puts every file
|
||||
back where it was. Only when nothing can be written beside the folder
|
||||
(a drive root, a read-only parent) does the holder go inside it, with
|
||||
no folder name, and the same copy puts the files back.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.Lock()
|
||||
self.stamp = ""
|
||||
# Holder -> the run folder this run created in it.
|
||||
self._runs: dict[Path, Path] = {}
|
||||
# Run folder -> [files, bytes] kept (or, under --dry-run, to keep).
|
||||
self.counts: dict[Path, list[int]] = {}
|
||||
|
||||
def reset(self) -> None:
|
||||
with self._lock:
|
||||
self.stamp = ""
|
||||
self._runs.clear()
|
||||
self.counts.clear()
|
||||
|
||||
def _stamp(self) -> str:
|
||||
if not self.stamp:
|
||||
self.stamp = time.strftime("%Y%m%d-%H%M%S")
|
||||
return self.stamp
|
||||
|
||||
@staticmethod
|
||||
def _holders(folder: Path) -> list[tuple[Path, str]]:
|
||||
beside = [(folder.parent / BACKUP_DIR, folder.name)] if (
|
||||
folder.name and folder.parent != folder) else []
|
||||
return beside + [(folder / BACKUP_DIR, "")]
|
||||
|
||||
def place(self, folder: Path, create: bool) -> tuple[Path, Path]:
|
||||
"""(run folder, folder the relative path goes under) for files of folder."""
|
||||
with self._lock:
|
||||
stamp = self._stamp()
|
||||
failure: OSError | None = None
|
||||
for holder, prefix in self._holders(folder):
|
||||
if not create:
|
||||
return holder / stamp, holder / stamp / prefix
|
||||
run = self._runs.get(holder)
|
||||
if run is None:
|
||||
try:
|
||||
holder.mkdir(parents=True, exist_ok=True)
|
||||
for n in range(1, 1000):
|
||||
run = holder / (stamp if n == 1 else f"{stamp}-{n}")
|
||||
try:
|
||||
run.mkdir()
|
||||
break
|
||||
except FileExistsError:
|
||||
continue
|
||||
except OSError as exc:
|
||||
failure = exc
|
||||
continue
|
||||
self._runs[holder] = run
|
||||
return run, run / prefix
|
||||
raise failure or OSError(errno.EEXIST, "no backup folder name left")
|
||||
|
||||
def plan(self, folder: Path, size: int) -> Path:
|
||||
"""Count a file a real run would keep, writing nothing. Returns its run folder."""
|
||||
run, _under = self.place(folder, create=False)
|
||||
with self._lock:
|
||||
tally = self.counts.setdefault(run, [0, 0])
|
||||
tally[0] += 1
|
||||
tally[1] += size
|
||||
return run
|
||||
|
||||
def keep(
|
||||
self, folder: Path, rel: PurePosixPath, dest: Path,
|
||||
expected: dict[str, str], fresh: Path | None = None,
|
||||
) -> None:
|
||||
"""Move the file at dest aside when its content is not the one coming.
|
||||
|
||||
Nothing is done for a missing file, a link (its target is not
|
||||
touched by the replace) or a file already holding the content, by
|
||||
the manifest's hashes or, without them, byte for byte against the
|
||||
fresh copy. Raises _NotKept when the file could not be set aside:
|
||||
the replace must not happen then.
|
||||
"""
|
||||
try:
|
||||
info = os.lstat(dest)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except OSError as exc:
|
||||
raise _NotKept(exc.strerror or str(exc)) from exc
|
||||
if not stat.S_ISREG(info.st_mode):
|
||||
return
|
||||
try:
|
||||
if expected:
|
||||
same = _digest_file(dest, tuple(expected)) == expected
|
||||
else:
|
||||
same = fresh is not None and _same_bytes(dest, fresh)
|
||||
except FileNotFoundError:
|
||||
return # another run replaced it meanwhile
|
||||
except OSError as exc:
|
||||
raise _NotKept(exc.strerror or str(exc)) from exc
|
||||
if same:
|
||||
return
|
||||
try:
|
||||
run, under = self.place(folder, create=True)
|
||||
kept = under / Path(*rel.parts)
|
||||
if os.path.lexists(kept):
|
||||
# This run kept the user's file at that path already: the
|
||||
# one in place now is one this run wrote.
|
||||
return
|
||||
kept.parent.mkdir(parents=True, exist_ok=True)
|
||||
_move_aside(dest, kept)
|
||||
except FileNotFoundError:
|
||||
if not os.path.lexists(dest):
|
||||
return # another run moved or replaced it meanwhile
|
||||
raise _NotKept("the backup folder could not be written") from None
|
||||
except OSError as exc:
|
||||
raise _NotKept(exc.strerror or str(exc)) from exc
|
||||
with self._lock:
|
||||
tally = self.counts.setdefault(run, [0, 0])
|
||||
tally[0] += 1
|
||||
tally[1] += info.st_size
|
||||
|
||||
def summary(self) -> list[dict]:
|
||||
"""[{folder, files, bytes}], one per run folder, for the report and events."""
|
||||
with self._lock:
|
||||
return [{"folder": str(run), "files": files, "bytes": size}
|
||||
for run, (files, size) in sorted(self.counts.items())]
|
||||
|
||||
|
||||
KEPT = KeptFiles()
|
||||
|
||||
|
||||
def _same_bytes(a: Path, b: Path) -> bool:
|
||||
if a.stat().st_size != b.stat().st_size:
|
||||
return False
|
||||
return _digest_file(a, ("sha1",)) == _digest_file(b, ("sha1",))
|
||||
|
||||
|
||||
def _move_aside(dest: Path, kept: Path) -> None:
|
||||
"""Move dest to kept: a rename on one drive, else a copy then a delete.
|
||||
|
||||
The copy goes under a temporary name first, so an interrupted copy
|
||||
never leaves a cut file under the kept name, and dest is only deleted
|
||||
once its copy is whole.
|
||||
"""
|
||||
try:
|
||||
os.rename(dest, kept)
|
||||
return
|
||||
except OSError:
|
||||
if not dest.is_file():
|
||||
raise
|
||||
scratch = kept.parent / f".{kept.name}.part"
|
||||
try:
|
||||
shutil.copy2(dest, scratch)
|
||||
os.replace(scratch, kept)
|
||||
except OSError:
|
||||
if os.path.lexists(scratch):
|
||||
os.unlink(scratch)
|
||||
raise
|
||||
dest.unlink()
|
||||
|
||||
|
||||
def _kept_slot(root: Path, value: object) -> tuple[Path, PurePosixPath]:
|
||||
"""The folder a manifest destination is written in, and its path below
|
||||
it, as _destination_path places it: a RetroDECK folder moved elsewhere
|
||||
is its own folder."""
|
||||
relative = _safe_relative_path(value, "dest")
|
||||
moved = _MOVED_FOLDERS.get(root, {})
|
||||
if len(relative.parts) > 1 and relative.parts[0] in moved:
|
||||
return moved[relative.parts[0]].resolve(), PurePosixPath(*relative.parts[1:])
|
||||
return root.resolve(), relative
|
||||
|
||||
|
||||
def plan_kept(entries: list[dict], root: Path) -> dict[Path, list[int]]:
|
||||
"""Count, under --dry-run, the files in place a real run would keep.
|
||||
|
||||
entries are files whose content check failed. Returns {run folder:
|
||||
[files, bytes]} for these entries, also counted in KEPT for the summary.
|
||||
"""
|
||||
planned: dict[Path, list[int]] = {}
|
||||
for entry in entries:
|
||||
try:
|
||||
dest = _destination_path(root, entry["dest"])
|
||||
info = os.lstat(dest)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if stat.S_ISREG(info.st_mode):
|
||||
run = KEPT.plan(_kept_slot(root, entry["dest"])[0], info.st_size)
|
||||
tally = planned.setdefault(run, [0, 0])
|
||||
tally[0] += 1
|
||||
tally[1] += info.st_size
|
||||
return planned
|
||||
|
||||
|
||||
def _kept_lines() -> list[str]:
|
||||
"""Where the files this run replaced are, and how to put them back."""
|
||||
lines = []
|
||||
for item in KEPT.summary():
|
||||
folder = Path(item["folder"])
|
||||
lines.append(f"Kept {_plural(item['files'], 'file')} you had "
|
||||
f"({format_size(item['bytes'])}), which the collection's "
|
||||
f"copies replaced, in {folder}")
|
||||
lines.append(f" To put them back: {_INVOCATION.launcher()} --restore "
|
||||
f"{_INVOCATION.word(str(folder))}")
|
||||
return lines
|
||||
|
||||
|
||||
def restore_kept(run: Path, dry_run: bool = False) -> int:
|
||||
"""Put every file of a backup run folder back where it was.
|
||||
|
||||
run is a <holder>/<date-time> folder a run wrote, its holder named
|
||||
retrobios-backup. Each file goes to the same path below the holder's
|
||||
parent, replacing what is there. The backup is left in place. Returns
|
||||
the exit code: 0 when every file went back, 1 when one failed, 2 when
|
||||
the folder is no backup.
|
||||
"""
|
||||
run = Path(os.path.abspath(os.path.expanduser(str(run))))
|
||||
if not run.is_dir() or run.parent.name != BACKUP_DIR:
|
||||
print(f"Error: {run} is not a folder the installer kept files in.", file=sys.stderr)
|
||||
print(f" Name one of the dated folders inside a {BACKUP_DIR} folder, as the "
|
||||
"end of the install printed it.", file=sys.stderr)
|
||||
return 2
|
||||
root = run.parent.parent
|
||||
files = sorted(
|
||||
path for path in run.rglob("*")
|
||||
if path.is_file() and not path.is_symlink()
|
||||
and not (path.name.startswith(".") and path.name.endswith(".part"))
|
||||
)
|
||||
if not files:
|
||||
print(f"{run} holds no file. Nothing to put back.")
|
||||
return 0
|
||||
restored = size = failed = 0
|
||||
for src in files:
|
||||
dest = root / src.relative_to(run)
|
||||
if dry_run:
|
||||
print(f" would put back {dest}")
|
||||
restored += 1
|
||||
size += src.stat().st_size
|
||||
continue
|
||||
try:
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
_clear_read_only(dest)
|
||||
_replace_with_copy(src, dest)
|
||||
except OSError as exc:
|
||||
print(f" failed: {dest} ({exc.strerror or exc})", file=sys.stderr)
|
||||
failed += 1
|
||||
continue
|
||||
restored += 1
|
||||
size += src.stat().st_size
|
||||
if dry_run:
|
||||
print(f"\nWould put back {_plural(restored, 'file')} ({format_size(size)}) "
|
||||
f"under {root}. Nothing was written.")
|
||||
return 0
|
||||
print(f"\nPut back {_plural(restored, 'file')} ({format_size(size)}) under {root}.")
|
||||
if failed:
|
||||
print(f"{_plural(failed, 'file')} could not be put back.", file=sys.stderr)
|
||||
print(f"The copies are still in {run}. Delete that folder once you no longer need it.")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def _download_one(
|
||||
f: dict, bios_path: Path, verbose: bool = False,
|
||||
progress: Progress | None = None,
|
||||
@@ -3558,6 +3833,9 @@ def _download_one(
|
||||
# written with sudo has to stay readable by the emulator's user.
|
||||
os.chmod(tmp_path, (0o777 if f.get("executable") else 0o666) & ~_UMASK)
|
||||
_clear_read_only(dest)
|
||||
# The file in place, a personal dump or another revision, is
|
||||
# moved aside first: replaced, it was gone for good.
|
||||
KEPT.keep(*_kept_slot(bios_path, f["dest"]), dest, expected, tmp_path)
|
||||
if partial is not None and _NO_RENAME_WHILE_OPEN:
|
||||
# Windows renames no open file. Elsewhere the lock is held
|
||||
# until the copy has its final name.
|
||||
@@ -3577,6 +3855,12 @@ def _download_one(
|
||||
tmp_path.unlink()
|
||||
return f["dest"], False, "stopped"
|
||||
|
||||
except _NotKept as exc:
|
||||
# No attempt changes it, and the file in place stays as it is.
|
||||
if partial is None and tmp_path is not None and tmp_path.exists():
|
||||
tmp_path.unlink()
|
||||
return f["dest"], False, f"{NOT_KEPT} ({exc})"
|
||||
|
||||
except _Corrupt:
|
||||
if verbose:
|
||||
print(f" {reason} on attempt {attempt}", file=sys.stderr)
|
||||
@@ -3800,6 +4084,9 @@ def _write_refused(exc: OSError) -> bool:
|
||||
# and whose attribute could not be cleared: replacing it is refused while its
|
||||
# folder stays writable.
|
||||
READ_ONLY = "file in place is read-only"
|
||||
# A file in place whose content differs, which could not be moved aside: it
|
||||
# is left as it is rather than lost.
|
||||
NOT_KEPT = "file in place could not be kept aside"
|
||||
|
||||
|
||||
def _clear_read_only(dest: Path) -> None:
|
||||
@@ -4046,8 +4333,11 @@ def _report_outcome(
|
||||
"as answered.")
|
||||
print(f"{_plural(up_to_date, 'file')} already up to date at {_locations(paths)}.")
|
||||
return
|
||||
kept = _kept_lines()
|
||||
if interrupted:
|
||||
print(f"\nStopped. {_plural(downloaded, 'file')} installed.")
|
||||
for line in kept:
|
||||
print(line)
|
||||
print("Run the same command again to carry on from here. Nothing is lost.")
|
||||
sys.exit(130)
|
||||
|
||||
@@ -4080,7 +4370,7 @@ def _report_outcome(
|
||||
print(f"Destination: {_locations(paths)}")
|
||||
else:
|
||||
print(f"Location that was being written to: {_locations(paths)}")
|
||||
for line in advice:
|
||||
for line in kept + advice:
|
||||
print(line)
|
||||
print("Run the same command again once that is resolved. "
|
||||
"Files already installed are kept.")
|
||||
@@ -4090,7 +4380,7 @@ def _report_outcome(
|
||||
print(f"\nFinished with errors. {_plural(downloaded, 'file')} installed, "
|
||||
f"{_plural(errors, 'file')} failed.")
|
||||
print(f"Location: {_locations(paths)}")
|
||||
for line in advice:
|
||||
for line in kept + advice:
|
||||
print(line)
|
||||
print("Run the same command again to retry only those.")
|
||||
sys.exit(1)
|
||||
@@ -4098,6 +4388,8 @@ def _report_outcome(
|
||||
print(f"\nDone. {_plural(downloaded, 'file')} installed, "
|
||||
f"{_plural(up_to_date, 'file')} already up to date.")
|
||||
print(f"Location: {_locations(paths)}")
|
||||
for line in kept:
|
||||
print(line)
|
||||
if omitted:
|
||||
for line in _omission_lines(omitted, omitted_entries or []):
|
||||
print(line)
|
||||
@@ -4188,6 +4480,7 @@ def _confirm(prompt: str) -> bool:
|
||||
def _print_plan(
|
||||
to_download: list[dict], bios_path: Path, term: Terminal,
|
||||
trees: list[tuple[dict, dict[str, dict]]] = (),
|
||||
kept: dict[Path, list[int]] | None = None,
|
||||
) -> None:
|
||||
"""What a real run would fetch, by system, and nothing written.
|
||||
|
||||
@@ -4222,6 +4515,9 @@ def _print_plan(
|
||||
for entry, needed in trees:
|
||||
print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, "
|
||||
f"from {entry['url']}")
|
||||
for folder, (count, size) in sorted((kept or {}).items()):
|
||||
print(f"\n Would keep {_plural(count, 'file')} you have ({format_size(size)}), "
|
||||
f"whose contents differ, in {folder}")
|
||||
print("\n Nothing was written. Run without --dry-run to install.")
|
||||
|
||||
|
||||
@@ -4412,11 +4708,13 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict:
|
||||
"""Rebuild one data tree from its source archive, file by file verified.
|
||||
|
||||
Returns counts: present, installed, changed (the source no longer holds
|
||||
the recorded bytes), absent (not in the archive), and error, the reason
|
||||
the archive could not be read.
|
||||
the recorded bytes), absent (not in the archive), error, the reason
|
||||
the archive could not be read, and not_kept, the (path, reason) of each
|
||||
file in place that differs and could not be moved aside, left as it is.
|
||||
"""
|
||||
root, needed, present = data_directory_state(entry, bios_path)
|
||||
counts = {"present": present, "installed": 0, "changed": 0, "absent": 0, "error": ""}
|
||||
counts = {"present": present, "installed": 0, "changed": 0, "absent": 0, "error": "",
|
||||
"not_kept": []}
|
||||
if not needed:
|
||||
return counts
|
||||
try:
|
||||
@@ -4450,6 +4748,14 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict:
|
||||
size += len(chunk)
|
||||
out.write(chunk)
|
||||
if digest.hexdigest() == meta["sha1"] and size == meta["size"]:
|
||||
slot = f"{entry['dest']}/{rel}" if entry.get("dest") else rel
|
||||
try:
|
||||
KEPT.keep(*_kept_slot(bios_path, slot), target,
|
||||
{"sha1": meta["sha1"]})
|
||||
except _NotKept as exc:
|
||||
Path(out.name).unlink(missing_ok=True)
|
||||
counts["not_kept"].append((rel, str(exc)))
|
||||
continue
|
||||
os.replace(out.name, target)
|
||||
counts["installed"] += 1
|
||||
else:
|
||||
@@ -4596,7 +4902,7 @@ def _install_standalone_folders(
|
||||
pending += len(to_download)
|
||||
continue
|
||||
if args.dry_run:
|
||||
_print_plan(to_download, folder, term)
|
||||
_print_plan(to_download, folder, term, kept=plan_kept(mismatched, folder))
|
||||
continue
|
||||
if not to_download:
|
||||
print(" All files up to date.")
|
||||
@@ -4618,7 +4924,8 @@ def _install_standalone_folders(
|
||||
if args.check:
|
||||
if pending:
|
||||
print("\n Run the same command without --check to fetch them; "
|
||||
"a file with a wrong hash is replaced.")
|
||||
"a file with a wrong hash is replaced, and the one in place "
|
||||
f"kept in {BACKUP_DIR} beside its folder.")
|
||||
sys.exit(1)
|
||||
return True
|
||||
if args.dry_run:
|
||||
@@ -4679,9 +4986,12 @@ def do_standalone_copies(
|
||||
continue
|
||||
|
||||
if "pattern" in entry:
|
||||
# A kept file is never a source, when the backup had to go
|
||||
# inside the BIOS folder.
|
||||
sources = [
|
||||
f for f in bios_path.rglob("*")
|
||||
if fnmatch(f.name, entry["pattern"]) and f.is_file()
|
||||
and BACKUP_DIR not in f.relative_to(bios_path).parts
|
||||
]
|
||||
else:
|
||||
# `source` is where this platform's manifest put the file, and `file`
|
||||
@@ -4708,6 +5018,14 @@ def do_standalone_copies(
|
||||
if failures is not None:
|
||||
failures.append((str(dest), OUTSIDE_TARGET))
|
||||
continue
|
||||
try:
|
||||
KEPT.keep(resolved_dir, PurePosixPath(src.name), dest, {}, src)
|
||||
except _NotKept as exc:
|
||||
print(f" ERROR: {dest}: {NOT_KEPT} ({exc})")
|
||||
failed += 1
|
||||
if failures is not None:
|
||||
failures.append((str(dest), f"{NOT_KEPT} ({exc})"))
|
||||
continue
|
||||
try:
|
||||
if linked:
|
||||
# Writing through a link already at the name would
|
||||
@@ -5436,6 +5754,13 @@ def main() -> None:
|
||||
help="write the run as JSON events on stdout, one per line, and the "
|
||||
"usual text on stderr (implies --no-input)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--restore",
|
||||
type=Path,
|
||||
metavar="FOLDER",
|
||||
help=f"put back the files an earlier run replaced and kept in FOLDER "
|
||||
f"(a dated folder inside {BACKUP_DIR})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
action="version",
|
||||
@@ -5464,6 +5789,20 @@ def main() -> None:
|
||||
or args.list_systems or args.list_cores or args.list_emulators
|
||||
):
|
||||
parser.error("--quiet applies to an install, not to --check, --dry-run or a listing")
|
||||
if args.restore is not None:
|
||||
# The folder says where each file goes: nothing else applies.
|
||||
alone = ("platform", "dest", "target", "every_board", "check", "list_platforms",
|
||||
"list_targets", "standalone_copies", "emulator", "installed_cores",
|
||||
"cores_dir", "system", "core", "region", "no_game_data", "list_systems",
|
||||
"list_cores", "romm_firmware", "list_emulators", "events", "quiet",
|
||||
"yes", "verbose")
|
||||
given = [name for name in alone if getattr(args, name)]
|
||||
if args.jobs != 8:
|
||||
given.append("jobs")
|
||||
if given:
|
||||
parser.error("--restore puts files back from a folder and takes no "
|
||||
f"--{given[0].replace('_', '-')}")
|
||||
sys.exit(restore_kept(args.restore, args.dry_run))
|
||||
if args.every_board and args.target:
|
||||
parser.error("--every-board installs every board; --target names one")
|
||||
if args.no_game_data and (args.list_platforms or args.list_targets):
|
||||
@@ -5635,6 +5974,7 @@ def _unknown_target(target: str, plat: str, targets: dict) -> None:
|
||||
|
||||
def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
"""One run of the installer, once the options are read."""
|
||||
KEPT.reset()
|
||||
term = Terminal(real_stdout, quiet=args.quiet)
|
||||
interactive = sys.stdin.isatty() and not args.no_input and not args.quiet
|
||||
listing = (args.list_targets or args.list_systems or args.list_cores
|
||||
@@ -6106,7 +6446,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
print(f" ... and {len(missing) - len(shown)} more missing "
|
||||
"(--verbose lists them all)")
|
||||
print(" Run the same command without --check to fetch them; "
|
||||
"a file with a wrong hash is replaced.")
|
||||
"a file with a wrong hash is replaced, and the one in place "
|
||||
f"kept in {BACKUP_DIR} beside the BIOS folder.")
|
||||
check_pending += len(to_download)
|
||||
else:
|
||||
print("\n All files up to date.")
|
||||
@@ -6128,9 +6469,13 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
|
||||
if args.dry_run:
|
||||
files_pending += len(to_download)
|
||||
_print_plan(to_download, bios_path, term, [
|
||||
(entry, data_directory_state(entry, bios_path)[1]) for entry in data_dirs
|
||||
])
|
||||
trees = [(entry, data_directory_state(entry, bios_path)[1]) for entry in data_dirs]
|
||||
# A data file in place with other contents is kept like a BIOS file.
|
||||
kept = plan_kept(mismatched + [
|
||||
{"dest": f"{entry['dest']}/{rel}" if entry.get("dest") else rel}
|
||||
for entry, needed in trees for rel in needed
|
||||
], bios_path)
|
||||
_print_plan(to_download, bios_path, term, trees, kept)
|
||||
continue
|
||||
|
||||
if to_download:
|
||||
@@ -6212,6 +6557,10 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
f"{counts['error']}", file=sys.stderr)
|
||||
failures.append((entry["ref"], counts["error"]))
|
||||
total_errors += 1
|
||||
for rel, reason in counts["not_kept"]:
|
||||
where = f"{entry['dest']}/{rel}" if entry.get("dest") else rel
|
||||
failures.append((where, f"{NOT_KEPT} ({reason})"))
|
||||
total_errors += 1
|
||||
|
||||
# Standalone copies
|
||||
if (
|
||||
@@ -6253,7 +6602,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
|
||||
pending=check_pending if args.check else files_pending,
|
||||
failed=[{"dest": dest, "reason": str(reason)} for dest, reason in failures],
|
||||
skipped=_skipped(omitted_seen), left_out=_left_out_events(left_out_seen),
|
||||
interrupted=total_interrupted,
|
||||
kept=KEPT.summary(), interrupted=total_interrupted,
|
||||
)
|
||||
# A check scripted as `--check && launch` reads the exit code only.
|
||||
if args.check and check_pending:
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ esac
|
||||
TEMP_INSTALLER=""
|
||||
TEMP_DIRECTORY=""
|
||||
DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
|
||||
DEFAULT_INSTALL_SHA256="6732924ac45080a397dbc41b6bb08eec544c1738c593e5d41a7ddeba6d9a356a"
|
||||
DEFAULT_INSTALL_SHA256="8bf9b5bf68dbde80025b1b4e1c18e31f43c016758df91be202613933f1de17d5"
|
||||
MAX_INSTALLER_BYTES=2097152
|
||||
|
||||
cleanup() {
|
||||
|
||||
@@ -134,6 +134,9 @@ def follow(stream: Iterable[str], out: TextIO, summary: TextIO) -> None:
|
||||
event.get("pending", 0), "1" if event.get("interrupted") else "0")
|
||||
for failure in event.get("failed") or []:
|
||||
record(summary, "failed", failure.get("dest", ""), failure.get("reason", ""))
|
||||
for kept in event.get("kept") or []:
|
||||
record(summary, "kept", kept.get("folder", ""), kept.get("files", 0),
|
||||
size_text(kept.get("bytes", 0)))
|
||||
for group in event.get("skipped") or []:
|
||||
record(summary, "skipped", group.get("label", ""), group.get("files", 0),
|
||||
group.get("required", 0))
|
||||
|
||||
@@ -370,7 +370,7 @@ rb_update() {
|
||||
|
||||
# What the update did. Prints back or quit.
|
||||
rb_screen_summary() {
|
||||
local kind a b c d shown=0 failed=0
|
||||
local kind a b c d shown=0 failed=0 kept=0
|
||||
local end="none" installed=0 up_to_date=0 pending=0 interrupted=0 error=""
|
||||
while IFS=$'\t' read -r kind a b c d; do
|
||||
case "$kind" in
|
||||
@@ -402,6 +402,10 @@ rb_screen_summary() {
|
||||
rb_entry "summary" "info:failed:$shown" "Failed: $a" "$b"
|
||||
fi
|
||||
;;
|
||||
kept)
|
||||
kept=$((kept + 1))
|
||||
rb_entry "summary" "info:kept:$kept" "Kept $(rb_plural "$b" file) you had ($c)" "The collection's copies replaced them. Yours are in $a. Copy what that folder holds into the folder above retrobios-backup to put them back."
|
||||
;;
|
||||
skipped)
|
||||
rb_entry "summary" "info:skipped:$a" "Not in the collection ($a): $(rb_plural "$b" file)" "Required: $c. The RetroBIOS collection does not hold these files, so they cannot be downloaded."
|
||||
;;
|
||||
|
||||
@@ -501,7 +501,8 @@ def generate_home(
|
||||
"## Is this safe?",
|
||||
"",
|
||||
"The installer writes only inside your BIOS folder, checks every file "
|
||||
"before saving it, and never touches games or saves. "
|
||||
"before saving it, keeps any file of yours it replaces in a "
|
||||
"retrobios-backup folder beside it, and never touches games or saves. "
|
||||
"[How the command is checked](wiki/faq.md#is-piping-the-installer-into-a-shell-safe).",
|
||||
"",
|
||||
"Laws on copying console firmware differ by country. "
|
||||
|
||||
@@ -414,8 +414,9 @@ TRUST_NOTE = (
|
||||
"checks the installer against a fingerprint it carries, then the installer "
|
||||
"downloads the files your setup needs and checks each one before saving it. "
|
||||
"It writes only inside your BIOS folder. A BIOS file whose contents differ "
|
||||
"from the expected one is replaced with a checked copy, and files it does "
|
||||
"not know are never moved or deleted. Games and saves are never touched."
|
||||
"from the expected one is replaced with a checked copy, and yours is kept in "
|
||||
"a retrobios-backup folder beside it. Files it does not know are never moved "
|
||||
"or deleted. Games and saves are never touched."
|
||||
)
|
||||
|
||||
|
||||
|
||||
+235
-2
@@ -4966,7 +4966,8 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
|
||||
installer = events[0]["installer"]
|
||||
self.assertEqual(events, self._head("check", installer) + [
|
||||
{"event": "summary", "mode": "check", "installed": 0, "up_to_date": 1,
|
||||
"pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False},
|
||||
"pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "kept": [],
|
||||
"interrupted": False},
|
||||
{"event": "end", "code": 1},
|
||||
])
|
||||
self.assertIn("2/4 present (1 verified, 1 wrong hash)", proc.stderr)
|
||||
@@ -4976,6 +4977,9 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
|
||||
proc, events = self._run()
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
installer = events[0]["installer"]
|
||||
kept = events[-2]["kept"]
|
||||
self.assertEqual(len(kept), 1, kept)
|
||||
self.assertEqual(Path(kept[0]["folder"]).parent, self.home / install.BACKUP_DIR)
|
||||
progress = [event for event in events if event["event"] == "progress"]
|
||||
for event in progress:
|
||||
self.assertEqual(list(event), ["event", *install.EVENT_KEYS["progress"]])
|
||||
@@ -4989,7 +4993,9 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
|
||||
{"event": "download", "platform": "retroarch", "files": 3, "bytes": 245},
|
||||
*done,
|
||||
{"event": "summary", "mode": "install", "installed": 3, "up_to_date": 1,
|
||||
"pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False},
|
||||
"pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [],
|
||||
"kept": [{"folder": kept[0]["folder"], "files": 1, "bytes": 11}],
|
||||
"interrupted": False},
|
||||
{"event": "end", "code": 0},
|
||||
])
|
||||
self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])
|
||||
@@ -5031,5 +5037,232 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
|
||||
self.assertIn(f"`{key}`", row, f"{kind}.{key}")
|
||||
|
||||
|
||||
class AFileYouHadIsKept(unittest.TestCase):
|
||||
"""A BIOS file whose contents differ was replaced and lost: a personal
|
||||
dump or another revision could not be brought back by deleting what the
|
||||
installer wrote. It is moved beside the BIOS folder first, never inside,
|
||||
where a core that finds its BIOS by content could take it."""
|
||||
|
||||
NEW = b"the collection's copy"
|
||||
OLD = b"a dump of my own"
|
||||
|
||||
def setUp(self):
|
||||
install.KEPT.reset()
|
||||
self.addCleanup(install.KEPT.reset)
|
||||
self.home = Path(_tempdir(self))
|
||||
self.bios = self.home / "system"
|
||||
self.bios.mkdir()
|
||||
|
||||
def _entry(self, dest: str) -> dict:
|
||||
return {"dest": dest, "repo_path": f"bios/{dest}", "size": len(self.NEW),
|
||||
"sha1": hashlib.sha1(self.NEW).hexdigest(),
|
||||
"sha256": hashlib.sha256(self.NEW).hexdigest()}
|
||||
|
||||
def _download(self, dest: str):
|
||||
with unittest.mock.patch.object(install.urllib.request, "urlopen",
|
||||
lambda *a, **k: _FakeResponse(self.NEW)):
|
||||
return install._download_one(self._entry(dest), self.bios)
|
||||
|
||||
def _run_folder(self) -> Path:
|
||||
runs = list((self.home / install.BACKUP_DIR).iterdir())
|
||||
self.assertEqual(len(runs), 1, runs)
|
||||
return runs[0]
|
||||
|
||||
def test_a_differing_file_is_kept_with_its_relative_path(self):
|
||||
(self.bios / "sub").mkdir()
|
||||
(self.bios / "sub" / "x.bin").write_bytes(self.OLD)
|
||||
self.assertEqual(self._download("sub/x.bin"), ("sub/x.bin", True, ""))
|
||||
self.assertEqual((self.bios / "sub" / "x.bin").read_bytes(), self.NEW)
|
||||
run = self._run_folder()
|
||||
self.assertEqual((run / "system" / "sub" / "x.bin").read_bytes(), self.OLD)
|
||||
self.assertEqual(install.KEPT.summary(),
|
||||
[{"folder": str(run), "files": 1, "bytes": len(self.OLD)}])
|
||||
# Nothing new sits inside the BIOS folder, where a scan would read it.
|
||||
self.assertEqual(sorted(p.name for p in self.bios.rglob("*")), ["sub", "x.bin"])
|
||||
|
||||
def test_an_identical_file_is_not_copied(self):
|
||||
(self.bios / "x.bin").write_bytes(self.NEW)
|
||||
self.assertEqual(self._download("x.bin"), ("x.bin", True, ""))
|
||||
self.assertFalse((self.home / install.BACKUP_DIR).exists())
|
||||
self.assertEqual(install.KEPT.summary(), [])
|
||||
|
||||
def test_a_missing_file_keeps_nothing(self):
|
||||
self.assertEqual(self._download("x.bin"), ("x.bin", True, ""))
|
||||
self.assertFalse((self.home / install.BACKUP_DIR).exists())
|
||||
|
||||
def test_a_second_replacement_keeps_the_first_file(self):
|
||||
# Two lists in one run writing one path: what is kept is the user's.
|
||||
(self.bios / "x.bin").write_bytes(self.OLD)
|
||||
self._download("x.bin")
|
||||
(self.bios / "x.bin").write_bytes(b"what this run wrote first")
|
||||
self._download("x.bin")
|
||||
self.assertEqual((self._run_folder() / "system" / "x.bin").read_bytes(), self.OLD)
|
||||
|
||||
def test_a_file_that_cannot_be_kept_stays(self):
|
||||
(self.bios / "x.bin").write_bytes(self.OLD)
|
||||
|
||||
def refuse(*_args):
|
||||
raise PermissionError(13, "Permission denied")
|
||||
|
||||
with unittest.mock.patch.object(install, "_move_aside", refuse):
|
||||
_dest, ok, reason = self._download("x.bin")
|
||||
self.assertFalse(ok)
|
||||
self.assertIn(install.NOT_KEPT, reason)
|
||||
self.assertEqual((self.bios / "x.bin").read_bytes(), self.OLD)
|
||||
self.assertEqual(sorted(p.name for p in self.bios.iterdir()), ["x.bin"])
|
||||
|
||||
def test_a_symlinked_subfolder_keeps_working(self):
|
||||
elsewhere = self.home / "shadPS4" / "sys_modules"
|
||||
elsewhere.mkdir(parents=True)
|
||||
(elsewhere / "lib.sprx").write_bytes(self.OLD)
|
||||
(self.bios / "shadps4").mkdir()
|
||||
(self.bios / "shadps4" / "sys_modules").symlink_to(elsewhere)
|
||||
self.assertTrue(self._download("shadps4/sys_modules/lib.sprx")[1])
|
||||
self.assertEqual((elsewhere / "lib.sprx").read_bytes(), self.NEW)
|
||||
self.assertTrue((self.bios / "shadps4" / "sys_modules").is_symlink())
|
||||
run = self._run_folder()
|
||||
kept = run / "system" / "shadps4" / "sys_modules" / "lib.sprx"
|
||||
self.assertEqual(kept.read_bytes(), self.OLD)
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
self.assertEqual(install.restore_kept(run), 0)
|
||||
self.assertEqual((elsewhere / "lib.sprx").read_bytes(), self.OLD)
|
||||
self.assertTrue((self.bios / "shadps4" / "sys_modules").is_symlink())
|
||||
|
||||
def test_a_link_at_the_name_is_not_kept(self):
|
||||
# The replace takes the link's place: the file it points to is untouched.
|
||||
target = self.home / "mine.bin"
|
||||
target.write_bytes(self.OLD)
|
||||
(self.bios / "x.bin").symlink_to(target)
|
||||
self.assertTrue(self._download("x.bin")[1])
|
||||
self.assertEqual(target.read_bytes(), self.OLD)
|
||||
self.assertFalse((self.home / install.BACKUP_DIR).exists())
|
||||
|
||||
@unittest.skipIf(hasattr(os, "geteuid") and os.geteuid() == 0, "root writes anywhere")
|
||||
def test_without_room_beside_the_folder_it_goes_inside(self):
|
||||
(self.bios / "x.bin").write_bytes(self.OLD)
|
||||
self.home.chmod(0o555)
|
||||
self.addCleanup(self.home.chmod, 0o755)
|
||||
self.assertTrue(self._download("x.bin")[1])
|
||||
runs = list((self.bios / install.BACKUP_DIR).iterdir())
|
||||
self.assertEqual(len(runs), 1)
|
||||
self.assertEqual((runs[0] / "x.bin").read_bytes(), self.OLD)
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
self.assertEqual(install.restore_kept(runs[0]), 0)
|
||||
self.assertEqual((self.bios / "x.bin").read_bytes(), self.OLD)
|
||||
|
||||
def test_a_data_tree_file_is_kept(self):
|
||||
import zipfile
|
||||
|
||||
tree = self.bios / "dolphin-emu" / "Sys"
|
||||
tree.mkdir(parents=True)
|
||||
(tree / "a.ini").write_bytes(self.OLD)
|
||||
archive = self.home / "src.zip"
|
||||
with zipfile.ZipFile(archive, "w") as zf:
|
||||
zf.writestr("Sys/a.ini", self.NEW)
|
||||
entry = {"ref": "d", "dest": "dolphin-emu", "url": "https://example.invalid/d.zip",
|
||||
"type": "zip", "files": {"Sys/a.ini": {
|
||||
"sha1": hashlib.sha1(self.NEW).hexdigest(), "size": len(self.NEW)}}}
|
||||
|
||||
def fetch(url, directory):
|
||||
copy = Path(directory) / "a.zip"
|
||||
shutil.copy(archive, copy)
|
||||
return copy
|
||||
|
||||
with unittest.mock.patch.object(install, "_fetch_archive", fetch):
|
||||
counts = install.install_data_directory(entry, self.bios)
|
||||
self.assertEqual(counts["installed"], 1)
|
||||
self.assertEqual((tree / "a.ini").read_bytes(), self.NEW)
|
||||
kept = self._run_folder() / "system" / "dolphin-emu" / "Sys" / "a.ini"
|
||||
self.assertEqual(kept.read_bytes(), self.OLD)
|
||||
|
||||
def test_a_standalone_copy_keeps_the_emulators_file(self):
|
||||
(self.bios / "ps2-0230a.bin").write_bytes(self.NEW)
|
||||
pcsx2 = self.home / "PCSX2" / "bios"
|
||||
pcsx2.mkdir(parents=True)
|
||||
(pcsx2 / "ps2-0230a.bin").write_bytes(self.OLD)
|
||||
manifest = {"standalone_copies": [
|
||||
{"pattern": "ps2-*.bin", "emulator": "pcsx2", "targets": {"linux": [str(pcsx2)]}}]}
|
||||
self.assertEqual(install.do_standalone_copies(manifest, self.bios, "linux"), (1, 0, 0))
|
||||
self.assertEqual((pcsx2 / "ps2-0230a.bin").read_bytes(), self.NEW)
|
||||
runs = list((self.home / "PCSX2" / install.BACKUP_DIR).iterdir())
|
||||
self.assertEqual((runs[0] / "bios" / "ps2-0230a.bin").read_bytes(), self.OLD)
|
||||
# Copied again, the file is the same: nothing more is kept.
|
||||
self.assertEqual(install.do_standalone_copies(manifest, self.bios, "linux"), (1, 0, 0))
|
||||
self.assertEqual(install.KEPT.summary()[0]["files"], 1)
|
||||
|
||||
def test_a_kept_file_inside_the_tree_is_no_copy_source(self):
|
||||
inside = self.bios / install.BACKUP_DIR / "20261022-153000"
|
||||
inside.mkdir(parents=True)
|
||||
(inside / "ps2-old.bin").write_bytes(self.OLD)
|
||||
pcsx2 = self.home / "PCSX2" / "bios"
|
||||
pcsx2.mkdir(parents=True)
|
||||
manifest = {"standalone_copies": [
|
||||
{"pattern": "ps2-*.bin", "emulator": "pcsx2", "targets": {"linux": [str(pcsx2)]}}]}
|
||||
self.assertEqual(install.do_standalone_copies(manifest, self.bios, "linux"), (0, 0, 0))
|
||||
|
||||
def test_restore_refuses_a_folder_that_is_no_backup(self):
|
||||
with contextlib.redirect_stderr(io.StringIO()) as err:
|
||||
self.assertEqual(install.restore_kept(self.bios), 2)
|
||||
self.assertIn(install.BACKUP_DIR, err.getvalue())
|
||||
|
||||
|
||||
class KeptFilesEndToEnd(unittest.TestCase, _SyntheticTree):
|
||||
"""The run says what it kept and where, --dry-run what it would keep
|
||||
and writes nothing, and --restore puts the original bytes back."""
|
||||
|
||||
def setUp(self):
|
||||
self.env = self.serve_tree(self)
|
||||
|
||||
def test_dry_run_says_what_would_be_kept_and_writes_nothing(self):
|
||||
proc = _install_run(self.env, "--platform", "retroarch", "--dest", str(self.bios),
|
||||
"--dry-run", "--no-input")
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
self.assertIn("Would keep 1 file you have (11 B)", proc.stdout)
|
||||
self.assertIn(str(self.home / install.BACKUP_DIR), proc.stdout)
|
||||
self.assertFalse((self.home / install.BACKUP_DIR).exists())
|
||||
self.assertEqual((self.bios / "b.bin").read_bytes(), b"other bytes")
|
||||
events = _events(_install_run(self.env, "--platform", "retroarch", "--dest",
|
||||
str(self.bios), "--dry-run", "--events"))
|
||||
kept = events[-2]["kept"]
|
||||
self.assertEqual([(item["files"], item["bytes"]) for item in kept], [(1, 11)])
|
||||
self.assertFalse((self.home / install.BACKUP_DIR).exists())
|
||||
|
||||
def test_install_then_restore(self):
|
||||
proc = _install_run(self.env, "--platform", "retroarch", "--dest", str(self.bios),
|
||||
"--no-input", "--jobs", "1")
|
||||
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
||||
self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])
|
||||
runs = list((self.home / install.BACKUP_DIR).iterdir())
|
||||
self.assertEqual(len(runs), 1)
|
||||
self.assertIn("Kept 1 file you had (11 B), which the collection's copies "
|
||||
f"replaced, in {runs[0]}", proc.stdout)
|
||||
self.assertIn(f"--restore {runs[0]}", proc.stdout)
|
||||
# a.bin held the expected bytes: it is not among the kept files.
|
||||
self.assertEqual([p.name for p in (runs[0] / "bios").iterdir()], ["b.bin"])
|
||||
|
||||
refused = _install_run(self.env, "--restore", str(runs[0]), "--platform", "retroarch")
|
||||
self.assertEqual(refused.returncode, 2)
|
||||
self.assertIn("--restore", refused.stderr)
|
||||
|
||||
looked = _install_run(self.env, "--restore", str(runs[0]), "--dry-run")
|
||||
self.assertEqual(looked.returncode, 0, looked.stderr)
|
||||
self.assertIn(f"would put back {self.bios / 'b.bin'}", looked.stdout)
|
||||
self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])
|
||||
|
||||
back = _install_run(self.env, "--restore", str(runs[0]))
|
||||
self.assertEqual(back.returncode, 0, back.stdout + back.stderr)
|
||||
self.assertEqual((self.bios / "b.bin").read_bytes(), b"other bytes")
|
||||
self.assertEqual((self.bios / "a.bin").read_bytes(), self.FILES["bios/A/a.bin"])
|
||||
self.assertIn("Put back 1 file", back.stdout)
|
||||
# The backup stays until the user deletes it.
|
||||
self.assertTrue((runs[0] / "bios" / "b.bin").is_file())
|
||||
|
||||
# A second install keeps the restored file in a new folder.
|
||||
again = _install_run(self.env, "--platform", "retroarch", "--dest", str(self.bios),
|
||||
"--no-input", "--jobs", "1")
|
||||
self.assertEqual(again.returncode, 0, again.stderr)
|
||||
self.assertEqual(len(list((self.home / install.BACKUP_DIR).iterdir())), 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+12
-5
@@ -141,7 +141,9 @@ class EventRecords(unittest.TestCase):
|
||||
json.dumps({"event": "summary", "mode": "install", "installed": 0,
|
||||
"up_to_date": 4, "pending": 1,
|
||||
"failed": [{"dest": "x.bin", "reason": "HTTP 404"}],
|
||||
"skipped": [], "left_out": [], "interrupted": False}),
|
||||
"skipped": [], "left_out": [],
|
||||
"kept": [{"folder": "/b/retrobios-backup/1", "files": 1,
|
||||
"bytes": 2048}], "interrupted": False}),
|
||||
json.dumps({"event": "end", "code": 2}),
|
||||
]
|
||||
out, summary = io.StringIO(), io.StringIO()
|
||||
@@ -149,7 +151,8 @@ class EventRecords(unittest.TestCase):
|
||||
self.assertEqual(out.getvalue().splitlines(), [
|
||||
"download\t1\t2 KB", "file\t1\t1\tfailed\tx.bin\t0 B\t2 KB"])
|
||||
self.assertEqual(summary.getvalue().splitlines(), [
|
||||
"summary\t0\t4\t1\t0", "failed\tx.bin\tHTTP 404", "end\t2"])
|
||||
"summary\t0\t4\t1\t0", "failed\tx.bin\tHTTP 404",
|
||||
"kept\t/b/retrobios-backup/1\t1\t2 KB", "end\t2"])
|
||||
|
||||
def test_a_system_id_that_is_no_slug_is_dropped(self):
|
||||
# The id becomes a bash array subscript in the launcher, where a
|
||||
@@ -228,7 +231,7 @@ if check:
|
||||
present=3 if done else 1, missing=0 if done else 1, wrong_hash=0 if done else 1,
|
||||
bytes_needed=0 if done else 300, data_missing=0, not_installable=1)
|
||||
emit("summary", mode="check", installed=0, up_to_date=3 if done else 1,
|
||||
pending=0 if done else 2, failed=[], skipped=[], left_out=[],
|
||||
pending=0 if done else 2, failed=[], skipped=[], left_out=[], kept=[],
|
||||
interrupted=False)
|
||||
end(0 if done else 1)
|
||||
emit("download", platform="retroarch", files=2, bytes=300)
|
||||
@@ -238,7 +241,8 @@ for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1):
|
||||
(state / "installed").write_text("")
|
||||
emit("summary", mode="install", installed=2, up_to_date=1, pending=0, failed=[],
|
||||
skipped=[{"reason": "not_found", "label": "not found", "files": 1, "required": 1}],
|
||||
left_out=[], interrupted=False)
|
||||
left_out=[], kept=[{"folder": "/roms/retrobios-backup/20261022-153000", "files": 1,
|
||||
"bytes": 100}], interrupted=False)
|
||||
end(0)
|
||||
'''
|
||||
|
||||
@@ -371,7 +375,8 @@ class TheScreens(unittest.TestCase):
|
||||
shown = self.run_port("missing", ["update", "back", "quit"])
|
||||
self.assertEqual(self.lists(shown), [
|
||||
("menu", ["status", "update", "choose", "check", "quit"]),
|
||||
("summary", ["info:result", "back", "quit", "info:skipped:not found"]),
|
||||
("summary", ["info:result", "back", "quit", "info:kept:1",
|
||||
"info:skipped:not found"]),
|
||||
("menu", ["status", "check", "quit"]),
|
||||
])
|
||||
menu = shown[[s[0] for s in shown].index("list")][2]
|
||||
@@ -385,6 +390,8 @@ class TheScreens(unittest.TestCase):
|
||||
])
|
||||
summary = [s for s in shown if s[:2] == ("list", "summary")][0][2]
|
||||
self.assertEqual(summary["info:result"][0], "Installed: 2 files")
|
||||
self.assertEqual(summary["info:kept:1"][0], "Kept 1 file you had (100 B)")
|
||||
self.assertIn("/roms/retrobios-backup/20261022-153000", summary["info:kept:1"][1])
|
||||
self.assertEqual(self.calls(), [
|
||||
"--check --events",
|
||||
"--events --platform retroarch",
|
||||
|
||||
+12
-5
@@ -51,9 +51,10 @@ Details: [All installer options](installer.md), with the full behaviour.
|
||||
|
||||
## Will the installer change my other files?
|
||||
|
||||
No. It writes only inside the BIOS folder, and games and saves are never
|
||||
No. It writes only inside the BIOS folder and `retrobios-backup` beside it, and games and saves are never
|
||||
touched. Inside that folder, a BIOS file whose contents differ from the
|
||||
expected one is replaced with a checked copy. Files the installer does not
|
||||
expected one is replaced with a checked copy, and your old file is moved to a
|
||||
`retrobios-backup` folder beside the BIOS folder. Files the installer does not
|
||||
know are never read, moved or deleted. Copies into the folders of standalone
|
||||
emulators happen only when you ask for them with `--standalone-copies`.
|
||||
|
||||
@@ -69,14 +70,20 @@ the same files.
|
||||
|
||||
Yes, and re-running is the normal way to update. A file already present with
|
||||
the expected hash is left untouched, one whose contents do not match is
|
||||
replaced by the verified copy, and files the manifest does not name are never
|
||||
replaced by the verified copy (the old one is kept), and files the manifest does not name are never
|
||||
read, moved or deleted. `--check` does the same inspection and exits without
|
||||
writing.
|
||||
|
||||
## Can I undo it?
|
||||
|
||||
Yes. Delete the files from your BIOS folder. The installer changed nothing
|
||||
outside that folder.
|
||||
Yes. A file you had that the installer replaced is kept in a dated folder
|
||||
inside `retrobios-backup`, beside the BIOS folder. The end of the run prints
|
||||
that folder and the command that puts the files back, the same command with
|
||||
`--restore` and the folder. Copying the contents of the dated folder into the folder that holds
|
||||
`retrobios-backup` does the same. Files that were not there before can be
|
||||
deleted from the BIOS folder. The installer changed nothing outside it, apart
|
||||
from the backup folder.
|
||||
[More on kept files](installer.md#files-you-already-had).
|
||||
|
||||
## Can I install onto an SD card or another machine's drive?
|
||||
|
||||
|
||||
@@ -187,7 +187,7 @@ Checking existing files...
|
||||
The files the collection does not hold are listed after it again, as long as
|
||||
they are absent from the folder. `0 wrong hash` is the line that matters. A number there means a file is present
|
||||
whose contents the platform will reject, and running the install again replaces
|
||||
it.
|
||||
it. Your old file is kept in `retrobios-backup`, beside the BIOS folder.
|
||||
|
||||
## When the screen says something else
|
||||
|
||||
@@ -258,4 +258,7 @@ may need a file that nobody has dumped yet.
|
||||
detects and what it is allowed to write
|
||||
|
||||
To remove what the installer wrote, delete the files from the BIOS folder.
|
||||
Nothing outside it was changed.
|
||||
A file you had that it replaced is in `retrobios-backup`, beside that folder:
|
||||
the end of the run printed the `--restore` command that puts it back
|
||||
([details](installer.md#files-you-already-had)). Nothing else outside the
|
||||
BIOS folder was changed.
|
||||
+51
-6
@@ -109,7 +109,8 @@ macOS.
|
||||
times, after 1, 2 then 4 seconds or the delay a `Retry-After` header asks.
|
||||
A file of 16 MB or more keeps its partial copy (`.<name>.part`) when a
|
||||
transfer stops, and the next attempt or the next run requests only the
|
||||
rest.
|
||||
rest. A file already there with other contents is moved aside first, see
|
||||
[Files you already had](#files-you-already-had).
|
||||
7. Copies into standalone emulator directories, only with `--standalone-copies`.
|
||||
|
||||
A RetroArch run into an empty folder printed this on 2026-10-21:
|
||||
@@ -130,6 +131,48 @@ ones, grouped by system with the emulators that read them, and point at the
|
||||
emulator pages. They do not change the exit code of `--check`, since no run of
|
||||
the installer can fetch them.
|
||||
|
||||
## Files you already had
|
||||
|
||||
A file in the BIOS folder whose contents differ from the expected one (another
|
||||
revision, a personal dump) is replaced, but never lost. Just before the
|
||||
verified copy takes its place, the old file is moved to a dated folder beside
|
||||
the BIOS folder, under the same path:
|
||||
|
||||
```
|
||||
~/.config/retroarch/system/scph5501.bin the verified copy
|
||||
~/.config/retroarch/retrobios-backup/20261022-153000/system/scph5501.bin yours
|
||||
```
|
||||
|
||||
A file with the expected contents is never copied, and nothing is kept when
|
||||
nothing is replaced. The end of the run says how many files were kept, their
|
||||
size, the folder, and the command that puts them back:
|
||||
|
||||
```
|
||||
Kept 1 file you had (512.0 KB), which the collection's copies replaced, in /home/me/.config/retroarch/retrobios-backup/20261022-153000
|
||||
To put them back: python3 install.py --restore /home/me/.config/retroarch/retrobios-backup/20261022-153000
|
||||
```
|
||||
|
||||
`--restore FOLDER` copies every file of that folder back to its place and
|
||||
leaves the folder as it is, to delete once the emulators start as before.
|
||||
Copying the contents of the dated folder into the folder that holds
|
||||
`retrobios-backup` does the same by hand. `--dry-run --restore FOLDER` lists
|
||||
what would go back. `--dry-run` alone counts the files an install would keep.
|
||||
|
||||
The folder is beside the BIOS folder, not inside it, because some emulators
|
||||
find a BIOS by its contents and not by its name. DuckStation, SwanStation and
|
||||
Rustation read every file of the BIOS folder, and BizHawk walks its firmware
|
||||
folder with every subfolder: an old copy anywhere inside could be picked
|
||||
instead of the new one. Only when nothing can be written beside the BIOS folder
|
||||
(a drive root, a parent folder that refuses writes) does `retrobios-backup` go
|
||||
inside it, and the same `--restore` puts the files back. A file that cannot be
|
||||
moved aside is not replaced: it stays, and the run reports it as failed with
|
||||
`file in place could not be kept aside`.
|
||||
|
||||
On one drive the old file is moved, not copied, so keeping it needs no space
|
||||
beyond what it already used. A link sitting at a file's name is not kept: the
|
||||
file it points to is never touched. Data trees and `--standalone-copies` keep
|
||||
their replaced files the same way, beside the folder they were in.
|
||||
|
||||
## Choosing what to install
|
||||
|
||||
By default the installer fetches everything the platform runs. At a terminal,
|
||||
@@ -277,6 +320,7 @@ what the options name, everything by default.
|
||||
| `--yes`, `-y` | Do not ask before a download of 1 GB or more |
|
||||
| `--quiet`, `-q` | Print only failures and the final summary. Implies `--no-input` |
|
||||
| `--events` | Write the run as JSON events on standard output, one object per line, and the usual text on standard error. Implies `--no-input`. See [Machine-readable events](#machine-readable-events) |
|
||||
| `--restore FOLDER` | Put back the files a run replaced and kept in FOLDER, a dated folder inside `retrobios-backup`. Takes no other option but `--dry-run` and `--no-input`. See [Files you already had](#files-you-already-had) |
|
||||
| `--version` | Print the installer's identity, the first 12 hex digits of the SHA-256 the bootstraps pin |
|
||||
| `--list-platforms` | Print the supported platforms and what was detected here |
|
||||
| `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries |
|
||||
@@ -290,7 +334,8 @@ code 1 when a file is missing or does not match and 0 when every one does. Files
|
||||
with a wrong hash are listed first and in full, then the first ten missing ones
|
||||
(`--verbose` lists them all). `--dry-run` counts the data trees in its total. Without
|
||||
it, a file already present with the expected hash is left untouched, and one
|
||||
whose contents do not match is re-downloaded and replaced by the verified copy.
|
||||
whose contents do not match is re-downloaded and replaced by the verified copy,
|
||||
the old one kept beside the BIOS folder.
|
||||
Files the list does not name are never read, moved or deleted.
|
||||
|
||||
## Machine-readable events
|
||||
@@ -313,7 +358,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is
|
||||
| `download` | `platform`, `files`, `bytes` | Before the downloads start |
|
||||
| `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive |
|
||||
| `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file. `reason` says why one failed, and is empty otherwise |
|
||||
| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. Not written for a listing |
|
||||
| `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `kept`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. `kept` lists `{folder, files, bytes}`, one per backup folder, for the files in place the run moved aside before replacing them (with `--dry-run`, would move), and is empty when none. Not written for a listing |
|
||||
| `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) |
|
||||
|
||||
An option refused before the run starts (`install.py: error:`, code 2) writes
|
||||
@@ -508,7 +553,7 @@ exits 1.
|
||||
Some files are read by a standalone emulator from its own data directory
|
||||
rather than from the platform's BIOS tree. `--standalone-copies` copies them
|
||||
there after the install, and only then: without the flag nothing is written
|
||||
outside the selected tree.
|
||||
outside the selected tree, apart from the files kept in `retrobios-backup`.
|
||||
|
||||
The manifest carries the list, fifteen entries on the six platforms that
|
||||
declare them (Batocera, EmuDeck, Recalbox, RetroArch, RetroBat, RetroDECK):
|
||||
@@ -741,8 +786,8 @@ fields, the installer refuses to narrow rather than install everything.
|
||||
| Code | Meaning |
|
||||
|------|---------|
|
||||
| `0` | everything asked for is installed or already up to date |
|
||||
| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, or `--check` found files to fetch |
|
||||
| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a `--dest` folder that does not exist was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board`. The message then starts with `install.py: error:` |
|
||||
| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, or `--restore` could not put a file back |
|
||||
| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a `--dest` folder that does not exist was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` |
|
||||
| `130` | stopped with Ctrl+C, or with Ctrl+C or Ctrl+D at a question. Run the same command again to carry on |
|
||||
| `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) |
|
||||
|
||||
|
||||
@@ -45,7 +45,8 @@ looks for the name. Common causes:
|
||||
- Corrupted download
|
||||
|
||||
Running the same command without `--check` replaces each file with a wrong
|
||||
hash by a verified copy. Each system page on the site also lists the expected
|
||||
hash by a verified copy, and keeps yours in `retrobios-backup` beside the BIOS
|
||||
folder ([putting it back](installer.md#files-you-already-had)). Each system page on the site also lists the expected
|
||||
hashes, to compare against your file's MD5 or SHA1 by hand.
|
||||
|
||||
### Wrong region BIOS
|
||||
|
||||
Reference in new issue
Block a user