diff --git a/install.ps1 b/install.ps1 index 93d937a5a..93fff7d90 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "ce18b5e213fde35341231cd79a9c9205f57f3f096827dc50e26db9487d1b5349" + $defaultInstallSha256 = "e1460704fe40e4c91b9c410ae119ceb1ece3b375d9a3dc6be00a96d5f5d348bf" # The pin above refuses any changed byte: this cap only stops a body that # cannot be the installer (a sign-in portal's page) before it is hashed. $maximumInstallerBytes = 8MB diff --git a/install.py b/install.py index f466236bd..873ac0fa7 100644 --- a/install.py +++ b/install.py @@ -6548,6 +6548,39 @@ def apply_record(args: argparse.Namespace, plat: str, folder: Path) -> dict | No return setup +# Windows marks a directory junction, and any other reparse point, with this +# attribute. Before Python 3.12 lstat does not report a junction as a link, +# and os.walk(followlinks=False) descends into it. +_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + + +def _is_link(info: os.stat_result) -> bool: + """Whether what lstat read is a link: a symbolic link, or on Windows a + junction or any other reparse point.""" + return stat.S_ISLNK(info.st_mode) or bool( + getattr(info, "st_file_attributes", 0) & _REPARSE_POINT) + + +def _tree(top: Path) -> list[tuple[Path, os.stat_result]]: + """Every entry below top with what lstat read of it, children before + their folder. A link (a junction too) is listed, never entered.""" + found: list[tuple[Path, os.stat_result]] = [] + try: + names = sorted(os.listdir(top)) + except OSError: + return found + for name in names: + path = top / name + try: + info = os.lstat(path) + except OSError: + continue + if stat.S_ISDIR(info.st_mode) and not _is_link(info): + found += _tree(path) + found.append((path, info)) + return found + + def _is_backup_run(run: Path) -> bool: """Whether run is a dated folder a run of the installer made: a real folder holding the marker, inside a retrobios-backup holder that is no @@ -6558,20 +6591,18 @@ def _is_backup_run(run: Path) -> bool: except OSError: return False return (stat.S_ISDIR(info.st_mode) and stat.S_ISDIR(holder.st_mode) + and not _is_link(info) and not _is_link(holder) and run.parent.name == BACKUP_DIR and stat.S_ISREG(marker.st_mode)) def _backup_files(run: Path) -> list[Path]: """The files a backup run folder holds, its marker and note aside. Links - inside it are never followed: os.walk does not descend into a linked - folder, and a linked file is skipped.""" + inside it, junctions included, are never followed nor put back.""" return sorted( - Path(base) / name - for base, _dirs, names in os.walk(run, followlinks=False) - for name in names - if not (Path(base) / name).is_symlink() - and not (name.startswith(".") and name.endswith(".part")) - and not (Path(base) == run and name in (BACKUP_MARKER, REMOVED_NOTE)) + path for path, info in _tree(run) + if stat.S_ISREG(info.st_mode) and not _is_link(info) + and not (path.name.startswith(".") and path.name.endswith(".part")) + and not (path.parent == run and path.name in (BACKUP_MARKER, REMOVED_NOTE)) ) @@ -6617,6 +6648,10 @@ def _record_after_restore(run: Path, put_back: list[Path]) -> str: path = _below(folder, item["dest"])[0] if path is not None and not _check_ours(path, item)[0]: back.append(item) + if not back and put_back: + # The files went back, but not where the record's folder is: the + # folder moved or was renamed since the removal. + return f"it is for {folder}, which does not hold the files put back" if not back: return "" return _update_record(folder, note["platform"], @@ -6637,6 +6672,33 @@ def _record_after_restore(run: Path, put_back: list[Path]) -> str: return next((problem for problem in problems if problem), "") +def _recorded_copy(run: Path, dest: Path) -> bool: + """Whether the file at dest is the installer's own copy, as the record + of its folder names it: the collection gives it again, so a restore + replaces it without keeping it.""" + found = _record_folder(run, dest) + if found is None: + return False + try: + info = os.lstat(dest) + except OSError: + return False + if not stat.S_ISREG(info.st_mode) or _is_link(info): + return False + folder, rel = found + items = [item for setup in ((load_record(folder).data or {}).get("setups") or {}).values() + for item in setup.get("files") or [] + if str(item.get("root") or "") in ("", "data") + and item["dest"].casefold() == rel.casefold() and item["size"] == info.st_size] + if not items: + return False + try: + sha1 = _digest_file(dest, ("sha1",))["sha1"] + except OSError: + return False + return any(item["sha1"] == sha1 for item in items) + + def _record_merge(setup: dict | None, files: list[dict], choice: list[str]) -> dict: """A setup with files given back and the choice it had before a removal.""" merged = {(str(item.get("root") or ""), item["dest"]): item @@ -6688,8 +6750,9 @@ def restore_kept(run: Path, dry_run: bool = False) -> int: run is a / folder a run wrote, its holder named retrobios-backup. Each file goes to the same path below the holder's - parent, replacing what is there, and the install record of its folder - is made to say so. The backup is left in place. Returns the exit code: + parent. A file there with other contents is moved into a new dated + backup first, never lost. The install record of its folder is made to + say so. The backup is left in place. Returns the exit code: 0 when every file went back, 1 when one failed, 2 when the folder is no backup. """ @@ -6706,8 +6769,10 @@ def restore_kept(run: Path, dry_run: bool = False) -> int: return 0 restored = size = failed = 0 put_back: list[Path] = [] + KEPT.reset() for src in files: - dest = root / src.relative_to(run) + rel = PurePosixPath(*src.relative_to(run).parts) + dest = root / Path(*rel.parts) length = src.stat().st_size if dry_run: print(f" would put back {dest}") @@ -6715,11 +6780,20 @@ def restore_kept(run: Path, dry_run: bool = False) -> int: size += length continue try: + sha1 = _digest_file(src, ("sha1",))["sha1"] dest.parent.mkdir(parents=True, exist_ok=True) + # A file at the place that is not the backup's copy (a newer one, + # one the user put there) is moved aside first, into a new dated + # backup, as an install keeps what it replaces. Its first level + # is the folder the backup's holder sits in. + holder = (root / rel.parts[0], PurePosixPath(*rel.parts[1:])) \ + if len(rel.parts) > 1 else (root, rel) + if not _recorded_copy(run, dest): + KEPT.keep(*holder, dest, {"sha1": sha1}) _clear_read_only(dest) _replace_with_copy(src, dest) - except OSError as exc: - reason = exc.strerror or str(exc) + except (OSError, _NotKept) as exc: + reason = getattr(exc, "strerror", None) or str(exc) print(f" failed: {dest} ({reason})", file=sys.stderr) EVENTS.emit("restored", folder=str(run), path=str(dest), size=length, ok=False, reason=reason) @@ -6736,6 +6810,10 @@ def restore_kept(run: Path, dry_run: bool = False) -> int: return 0 problem = _record_after_restore(run, put_back) print(f"\nPut back {_plural(restored, 'file')} ({format_size(size)}) under {root}.") + for item in KEPT.summary(): + # The files that sat at those places, never lost. + print(f"Kept {_plural(item['files'], 'file')} that were in their place " + f"({format_size(item['bytes'])}) in {item['folder']}") if failed: print(f"{_plural(failed, 'file')} could not be put back.", file=sys.stderr) if problem: @@ -6759,6 +6837,8 @@ REMOVAL_REASONS = { "unreadable": "could not be read", "not_recorded": "not installed by RetroBIOS", } +# Why a planned file stays when it changed between the plan and its removal. +CHANGED_SINCE_PLAN = "changed since the plan" # Written into a backup that a removal made: the setup, its folder, its # choice before and the record's entries, so --restore gives them back. REMOVED_NOTE = ".retrobios-removed.json" @@ -6797,22 +6877,24 @@ def _entry_readers(entry: dict) -> set[str] | None: def _served(manifest: dict, fold: bool) -> dict[tuple[str, str], tuple]: """What the list serves, by (root, dest): (the sha1s it serves there, - its systems, its readers). A data tree's member counts under root data, - as the record keeps it.""" + its systems, the cores that read it, whether the platform's own list + names it). A place listed twice keeps both: a core's copy beside the + platform's own never makes the file a core's alone. A data tree's member + counts under root data, as the record keeps it.""" served: dict[tuple[str, str], tuple] = {} for entry in [*(manifest.get("files") or []), *(manifest.get("case_variants") or [])]: dest = str(entry.get("dest")) place = (str(entry.get("root") or ""), dest.casefold() if fold else dest) - sha1s, systems, readers = served.get(place, (set(), [], None)) + sha1s, systems, readers, own = served.get(place, (set(), [], set(), False)) mine = _entry_readers(entry) served[place] = (sha1s | _entry_sha1s(entry), sorted(set(systems) | set(entry_systems(entry))), - mine if readers is None else readers | (mine or set())) + readers | (mine or set()), own or mine is None) for tree in manifest.get("data_directories") or []: for rel, meta in (tree.get("files") or {}).items(): dest = f"{tree['dest']}/{rel}" if tree.get("dest") else rel served[("data", dest.casefold() if fold else dest)] = ( - {str(meta.get("sha1"))}, list(tree.get("systems") or []), None) + {str(meta.get("sha1"))}, list(tree.get("systems") or []), set(), True) return served @@ -6833,7 +6915,7 @@ def _below(folder: Path, dest: str) -> tuple[Path | None, str]: return None, "gone" except OSError: return None, "unreadable" - if stat.S_ISLNK(info.st_mode): + if _is_link(info): return None, "link" if not stat.S_ISDIR(info.st_mode): return None, "not_a_file" @@ -6849,7 +6931,7 @@ def _check_ours(path: Path, item: dict) -> tuple[str, os.stat_result | None]: return "gone", None except OSError: return "unreadable", None - if stat.S_ISLNK(info.st_mode): + if _is_link(info): return "link", info if not stat.S_ISREG(info.st_mode): return "not_a_file", info @@ -6925,16 +7007,23 @@ def plan_removal( else set(known["cores"]) | set(manifest.get("core_names") or {}) ) - chosen_cores remove, kept, gone, picked = [], [], [], set() + planned: set[tuple[str, str]] = set() for item in setup.get("files") or []: root, dest = str(item.get("root") or ""), str(item["dest"]) - sha1s, item_systems, readers = served.get((root, norm(dest)), (set(), [], None)) + sha1s, item_systems, readers, own = served.get( + (root, norm(dest)), (set(), [], set(), False)) if chosen_systems and not (chosen_systems & set(item_systems) or ( not item_systems and OTHER_SYSTEMS in chosen_systems)): continue - if chosen_cores and not (readers and readers & chosen_cores): + # The platform's own list keeps its files whatever core is removed. + if chosen_cores and (own or not readers & chosen_cores): continue if files and norm(dest) not in wanted: continue + if (root, norm(dest)) in planned: + # Two entries the drive opens as one file: planned once. + continue + planned.add((root, norm(dest))) picked.add(norm(dest)) entry = {"dest": dest, "root": root, "sha1": item["sha1"], "size": item["size"], "systems": item_systems} @@ -6961,7 +7050,7 @@ def plan_removal( if why or path is None or info is None: kept.append({**entry, "reason": why or "unreadable", "what": what}) continue - remove.append({**entry, "path": path, "inode": (info.st_dev, info.st_ino)}) + remove.append({**entry, "path": path, "inode": _stamp(info)}) kept += [{"dest": dest, "root": "", "sha1": "", "size": 0, "systems": [], "reason": "not_recorded", "what": ""} for dest in files if norm(dest) not in picked] @@ -7019,12 +7108,22 @@ def kept_line(item: dict) -> str: return f"{item['dest']}: {why}" +def _stamp(info: os.stat_result) -> tuple[int, int, int, int]: + """What tells a file from itself edited: device, inode, size, mtime.""" + return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns + + +class _ChangedSincePlan(OSError): + """The file is not the one the plan read: it stays.""" + + def remove_files(removal: Removal, setup: dict, backup: bool ) -> tuple[list[dict], list[dict], Path | None]: """Delete each file of the plan, or with backup move it into a new - backup folder: (removed, failed {dest, reason}, the backup folder). Each - file is read again just before: one that changed kind or inode since - the plan stays.""" + backup folder: (removed, failed {dest, reason, changed}, the backup + folder). Each file is read again just before, and hashed again: one + edited in place during the question, swapped, or turned into a link + stays, as changed since the plan.""" removed, failed = [], [] run: Path | None = None under: Path | None = None @@ -7033,9 +7132,10 @@ def remove_files(removal: Removal, setup: dict, backup: bool kept: Path | None = None try: info = os.lstat(path) - if (not stat.S_ISREG(info.st_mode) - or (info.st_dev, info.st_ino) != entry["inode"]): - raise OSError(errno.ESTALE, "changed since it was read") + if (not stat.S_ISREG(info.st_mode) or _is_link(info) + or _stamp(info) != entry["inode"] + or _digest_file(path, ("sha1",))["sha1"] != entry["sha1"]): + raise _ChangedSincePlan(errno.ESTALE, CHANGED_SINCE_PLAN) if backup: if run is None or under is None: run, under = KEPT.place(removal.folder, create=True) @@ -7048,7 +7148,8 @@ def remove_files(removal: Removal, setup: dict, backup: bool path.unlink() except OSError as exc: reason = exc.strerror or str(exc) - failed.append({"dest": entry["dest"], "reason": reason}) + failed.append({"dest": entry["dest"], "reason": reason, + "changed": isinstance(exc, _ChangedSincePlan)}) EVENTS.emit("removed", platform=removal.plat, dest=entry["dest"], size=entry["size"], ok=False, reason=reason, moved_to="") continue @@ -7071,7 +7172,7 @@ def _write_removed_note(run: Path, removal: Removal, setup: dict) -> None: def _record_after_removal(removal: Removal, removed: list[dict]) -> str: """The setup's record without the files removed and those gone, with the - choice left: the setup goes once nothing of it is chosen or held.""" + choice left: the setup goes once it holds no file.""" taken = {(entry["root"], entry["dest"]) for entry in removed} | { (str(item.get("root") or ""), item["dest"]) for item in removal.gone} @@ -7080,9 +7181,13 @@ def _record_after_removal(removal: Removal, removed: list[dict]) -> str: return None files = [item for item in setup.get("files") or [] if (str(item.get("root") or ""), item["dest"]) not in taken] - if removal.choice is None or not files: + if not files: return None - return {**setup, "written": _record_now(), "choice": list(removal.choice), + # A file that stayed (it failed, changed, or something still needs + # it) keeps its entry. With nothing left chosen, the choice stays the + # one recorded: no choice would read as the whole list. + choice = setup.get("choice") if removal.choice is None else removal.choice + return {**setup, "written": _record_now(), "choice": list(choice or []), "files": files} return _update_record(removal.folder, removal.plat, change) @@ -7114,6 +7219,15 @@ def run_removal(args: argparse.Namespace) -> int: moved with --backup), and the record made to say so. 0 done, 1 a file or the record failed, 2 nothing done.""" plat, folder = _action_setup(args) + if not os.path.isdir(folder): + # An unmounted card looks like every file gone: nothing is touched. + message = (f"Nothing was removed: {shown_path(folder)} is not there (a card " + "not mounted?).") + print(message, file=sys.stderr) + EVENTS.emit("removal_plan", platform=plat, folder=str(folder), files=0, bytes=0, + systems=[], dests=[], kept=[], gone=0, backup=bool(args.backup), + message=message) + return 2 read = load_record(folder) setup, _path, problem = setup_record(plat, folder) if setup is None or read.data is None: @@ -7181,8 +7295,7 @@ def run_removal(args: argparse.Namespace) -> int: for rel in by_hand: print(f" {rel}") if not removal.remove: - if removal.gone and not args.dry_run: - _record_after_removal(removal, []) + # The record keeps what it says until a removal is confirmed. return 2 if args.dry_run: print("Nothing was removed (--dry-run).") @@ -7211,6 +7324,13 @@ def run_removal(args: argparse.Namespace) -> int: message=f"Moved {_plural(len(removed), 'file')} here.") elif removed: print(f"Removed {_plural(len(removed), 'file')}, {format_size(freed)} freed.") + changed = [item for item in failed if item["changed"]] + if changed: + print(f"{agree(len(changed), 'file stays', 'files stay')}: " + f"{len(changed)} {CHANGED_SINCE_PLAN}.") + for item in changed: + print(f" {item['dest']}: {CHANGED_SINCE_PLAN}") + failed = [item for item in failed if not item["changed"]] for item in failed: print(f" failed: {item['dest']} ({item['reason']})", file=sys.stderr) if problem: @@ -7316,19 +7436,18 @@ def run_delete_backup(args: argparse.Namespace) -> int: print("Nothing was deleted.") return QUIT_CODE failed = 0 - # Bottom up, never through a link: a link goes, not what it names. - for base, dirs, names in os.walk(run, topdown=False, followlinks=False): - for name in [*names, *dirs]: - path = Path(base) / name - try: - if stat.S_ISDIR(os.lstat(path).st_mode): - path.rmdir() - else: - _clear_read_only(path) - path.unlink() - except OSError as exc: - failed += 1 - print(f" failed: {path} ({exc.strerror or exc})", file=sys.stderr) + # Bottom up, never through a link: a link goes, not what it names. A + # junction is removed as a folder, which leaves its target alone. + for path, info in _tree(run): + try: + if stat.S_ISDIR(info.st_mode): + path.rmdir() + else: + _clear_read_only(path) + path.unlink() + except OSError as exc: + failed += 1 + print(f" failed: {path} ({exc.strerror or exc})", file=sys.stderr) try: run.rmdir() except OSError as exc: @@ -25754,7 +25873,7 @@ def tui_removal_systems(state: TuiState) -> list[tuple[str, int, int]]: held: dict[str, list[int]] = {} for item in state.record.get("files") or []: systems = served.get((str(item.get("root") or ""), str(item["dest"])), - (set(), [], None))[1] + (set(), [], set(), False))[1] for system in systems or [OTHER_SYSTEMS]: tally = held.setdefault(system, [0, 0]) tally[0] += 1 diff --git a/install.sh b/install.sh index fbcfeff31..47dcfa96b 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="ce18b5e213fde35341231cd79a9c9205f57f3f096827dc50e26db9487d1b5349" +DEFAULT_INSTALL_SHA256="e1460704fe40e4c91b9c410ae119ceb1ece3b375d9a3dc6be00a96d5f5d348bf" # The pin above refuses any changed byte: this cap only stops a body that # cannot be the installer (a sign-in portal's page) before it is hashed. MAX_INSTALLER_BYTES=8388608 diff --git a/install/launchers/index.json b/install/launchers/index.json index cf4950b2f..583697cdb 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1280415, - "sha256": "ce18b5e213fde35341231cd79a9c9205f57f3f096827dc50e26db9487d1b5349", + "size": 1285922, + "sha256": "e1460704fe40e4c91b9c410ae119ceb1ece3b375d9a3dc6be00a96d5f5d348bf", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1334040, - "sha256": "5f526e2d068764c5a178eccaf82f97012ac8bda61a0db251501580a8efd698eb" + "size": 1339547, + "sha256": "974a225380c719ecee6dbc226ed17d1709ed406d08556657c15cccebb1eb587c" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1280415, - "sha256": "ce18b5e213fde35341231cd79a9c9205f57f3f096827dc50e26db9487d1b5349", + "size": 1285922, + "sha256": "e1460704fe40e4c91b9c410ae119ceb1ece3b375d9a3dc6be00a96d5f5d348bf", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1280415, - "sha256": "ce18b5e213fde35341231cd79a9c9205f57f3f096827dc50e26db9487d1b5349", + "size": 1285922, + "sha256": "e1460704fe40e4c91b9c410ae119ceb1ece3b375d9a3dc6be00a96d5f5d348bf", "executable": false }, { diff --git a/tests/test_install_removal.py b/tests/test_install_removal.py index 56c454f30..e14a1a9fd 100644 --- a/tests/test_install_removal.py +++ b/tests/test_install_removal.py @@ -20,6 +20,7 @@ import shutil import subprocess import sys import tempfile +import types import unittest from pathlib import Path from unittest import mock @@ -193,6 +194,9 @@ class NeverTouched(unittest.TestCase): self.assertIn("1 file stays: 1 changed since it was installed.", proc.stdout) self.assertIn("scph5501.bin: changed since it was installed", proc.stdout) self.assertIn("1 other file stays, not installed by RetroBIOS.", proc.stdout) + # The file that stayed keeps its entry: the setup is not dropped. + self.assertEqual([item["dest"] for item in repo.setup()["files"]], + ["scph5501.bin"]) def test_a_link_in_place_of_a_file_stays_and_its_target_too(self): with SharedRepository() as repo: @@ -466,5 +470,179 @@ class RestoreAnInstallBackup(unittest.TestCase): self.assertNotIn("gba_bios.bin", [item["dest"] for item in repo.setup()["files"]]) +class ReviewOfTheRemoval(unittest.TestCase): + """What a review of the first version found, each held by a test.""" + + def setUp(self): + self.root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, self.root, True) + self.folder = self.root / "bios" + self.folder.mkdir() + + def test_restore_keeps_the_file_in_place_aside(self): + with SharedRepository() as repo: + repo.install_both() + proc = repo.run("--remove", "--system", "nintendo-gba", "--backup", "--yes") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + [backup] = install.list_backups(repo.dest) + (repo.dest / "gba_bios.bin").write_bytes(b"a newer one the user put") + proc = repo.run("--restore", str(backup["folder"])) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual((repo.dest / "gba_bios.bin").read_bytes(), b"gba bios") + self.assertIn("Kept 1 file that were in their place", proc.stdout) + kept = [item for item in install.list_backups(repo.dest) + if item["folder"] != backup["folder"]] + self.assertEqual([item["files"] for item in kept], [1]) + [saved] = [path for path in kept[0]["folder"].rglob("gba_bios.bin")] + self.assertEqual(saved.read_bytes(), b"a newer one the user put") + # The same bytes in place: nothing more is kept. + again = repo.run("--restore", str(backup["folder"])) + self.assertEqual(again.returncode, 0, again.stdout + again.stderr) + self.assertNotIn("Kept", again.stdout) + + def _plan(self, payload: bytes, manifest: dict | None = None) -> tuple: + (self.folder / "a.bin").write_bytes(payload) + record = _record(self.folder, [_item("a.bin", payload)]) + return record, install.plan_removal( + "retroarch", self.folder, record, manifest or _served(("a.bin", payload))) + + def test_a_file_edited_in_place_after_the_plan_stays(self): + record, removal = self._plan(b"ours") + path = self.folder / "a.bin" + before = os.stat(path) + with open(path, "r+b") as handle: + handle.write(b"user") + # The same inode, size and even time: only its bytes tell. + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + removed, failed, _run = install.remove_files(removal, record["setups"]["retroarch"], + False) + self.assertEqual(removed, []) + self.assertEqual([(item["reason"], item["changed"]) for item in failed], + [(install.CHANGED_SINCE_PLAN, True)]) + self.assertEqual(path.read_bytes(), b"user") + + def test_a_file_that_failed_keeps_its_entry(self): + with SharedRepository() as repo: + repo.install_both() + if hasattr(os, "geteuid") and os.geteuid() == 0: + self.skipTest("root deletes anywhere") + (repo.dest / "pcsx").chmod(0o555) + try: + proc = repo.run("--remove", "--yes") + finally: + (repo.dest / "pcsx").chmod(0o755) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertEqual([item["dest"] for item in repo.setup()["files"]], + ["pcsx/extra.bin"]) + + def test_an_unmounted_folder_or_a_refusal_leaves_the_record(self): + with SharedRepository() as repo: + repo.install_both() + record = repo.record.read_bytes() + for path in repo.dest.rglob("*"): + if path.is_file(): + path.unlink() + asked = repo.run("--remove") + self.assertEqual(asked.returncode, 2) + self.assertEqual(repo.record.read_bytes(), record) + nothing = repo.run("--remove", "--system", "nintendo-gba", "--yes") + self.assertEqual(nothing.returncode, 2) + self.assertEqual(repo.record.read_bytes(), record) + shutil.rmtree(repo.dest) + gone = repo.run("--remove", "--yes") + self.assertEqual(gone.returncode, 2) + self.assertEqual(gone.stderr.strip().count("\n"), 0, gone.stderr) + self.assertIn("is not there", gone.stderr) + self.assertEqual(repo.record.read_bytes(), record) + + def junction(self, path: Path): + """lstat as Windows reads a directory junction at path, which Python + before 3.12 does not report as a link.""" + real = os.lstat + + def lstat(target, *args, **kwargs): + info = real(target, *args, **kwargs) + if Path(target) != path: + return info + return types.SimpleNamespace(**{name: getattr(info, name) for name in ( + "st_mode", "st_ino", "st_dev", "st_size", "st_mtime_ns", "st_uid")}, + st_file_attributes=0x400) + + return mock.patch.object(install.os, "lstat", side_effect=lstat) + + def test_a_junction_is_a_link(self): + (self.folder / "linked").mkdir() + payload = b"elsewhere" + (self.folder / "linked" / "deep.bin").write_bytes(payload) + record = _record(self.folder, [_item("linked/deep.bin", payload)]) + with self.junction(self.folder / "linked"): + removal = install.plan_removal("retroarch", self.folder, record, + _served(("linked/deep.bin", payload))) + self.assertEqual([item["reason"] for item in removal.kept], ["link"]) + + def test_a_backup_holding_a_junction_is_never_entered(self): + holder = self.root / install.BACKUP_DIR + run = holder / "20261103-120000" + (run / "bios").mkdir(parents=True) + (run / install.BACKUP_MARKER).write_text("marker") + (run / "bios" / "x.bin").write_bytes(b"kept") + junction = run / "bios" / "elsewhere" + junction.mkdir() + (junction / "precious.bin").write_bytes(b"not the backup's") + with self.junction(junction): + self.assertEqual(install._backup_files(run), [run / "bios" / "x.bin"]) + args = types.SimpleNamespace(delete_backup=run, dry_run=False, yes=True, + no_input=True) + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + install.run_delete_backup(args) + self.assertEqual((junction / "precious.bin").read_bytes(), b"not the backup's") + self.assertFalse((run / "bios" / "x.bin").exists()) + + def test_a_core_never_takes_a_file_the_platform_lists_too(self): + payload = b"bios" + manifest = {"files": [ + {"dest": "a.bin", "sha1": hashlib.sha1(payload).hexdigest(), "size": 4, + "cores": ["mgba"], "systems": ["nintendo-gba"]}, + {"dest": "a.bin", "sha1": hashlib.sha1(payload).hexdigest(), "size": 4, + "cores": None, "systems": ["nintendo-gba"]}]} + (self.folder / "a.bin").write_bytes(payload) + record = _record(self.folder, [_item("a.bin", payload)]) + removal = install.plan_removal("retroarch", self.folder, record, manifest, + cores=["mgba"]) + self.assertEqual(removal.remove, []) + self.assertTrue((self.folder / "a.bin").exists()) + + def test_two_names_one_file_are_planned_once(self): + payload = b"bios" + (self.folder / "A.BIN").write_bytes(payload) + record = _record(self.folder, [_item("A.BIN", payload), _item("a.bin", payload)]) + # The drive opens a.bin as A.BIN, as FAT does. + found = ("", os.lstat(self.folder / "A.BIN")) + with mock.patch.object(install, "keeps_case", return_value=False), \ + mock.patch.object(install, "_check_ours", return_value=found): + removal = install.plan_removal("retroarch", self.folder, record, + _served(("a.bin", payload))) + self.assertEqual([entry["dest"] for entry in removal.remove], ["A.BIN"]) + + def test_a_restore_into_a_moved_folder_says_the_record_was_not_updated(self): + with SharedRepository() as repo: + repo.install_both() + repo.run("--remove", "--system", "nintendo-gba", "--backup", "--yes") + [backup] = install.list_backups(repo.dest) + moved = repo.root.with_name(repo.root.name + "-moved") + repo.root.rename(moved) + try: + run = moved / install.BACKUP_DIR / backup["folder"].name + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()) as err: + code = install.restore_kept(run) + self.assertEqual(code, 1) + self.assertIn("The install record could not be updated", err.getvalue()) + self.assertTrue((moved / "dest" / "gba_bios.bin").is_file()) + finally: + moved.rename(repo.root) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index 05cf95281..0db0fdb6d 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -771,7 +771,12 @@ Kept 1 file already there (512.0 KB), which the collection's copies replaced, in ``` `--restore FOLDER` copies every file of that folder back to its place and -leaves the folder as it is, to delete once the emulators start as before. +leaves the folder as it is, to delete once the emulators start as before. A +file at that place with other contents (a newer one, one put there by hand) is +first moved into a new dated backup, as an install keeps what it replaces, and +the end of the run names that folder. The installer's own copy, as the +install record names it, is replaced without one: the collection gives it +again. Copying the contents of the dated folder into the folder that holds `retrobios-backup` does the same by hand. `--dry-run --restore FOLDER` lists what would go back. `--dry-run` alone counts the files an install would keep. @@ -838,20 +843,26 @@ release), and a file still needed by what stays. A file two systems share stays while one of them is still chosen, and a file another setup of the same folder recorded stays too. Removing everything also counts the files the record never names, added by hand or by another program, and leaves them. -A record that cannot be read is said in one line, and nothing is removed. +A record that cannot be read is said in one line, and nothing is removed. So +is a BIOS folder that is not there (a card not mounted), with code 2. Each +file is read and hashed again just before it goes: one edited during the +question stays, as changed since the plan. On FAT and exFAT cards, which open a name whatever its case, names are compared without case. -The record is updated after the run: the files removed leave it, the systems -or cores removed leave its choice, and a setup with nothing left goes from -it. `--restore` on a removal's backup gives the files back to the record with -the choice they had. On a backup of files an install replaced, the record lets +The record is updated only after the yes, and only for what was done: the +files removed leave it, the systems or cores removed leave its choice, a file +that stayed (changed, failed, still needed) keeps its entry, and a setup with +no file left goes from it. `--restore` on a removal's backup gives the files back to the record with +the choice they had. A folder moved since the removal is said, the record +left as it is. On a backup of files an install replaced, the record lets go of each file put back: it is the user's again. `--backups` lists the backups of the BIOS folder, newest first, and `--delete-backup FOLDER` deletes one after a question naming its date and size. Only a dated folder holding `.retrobios-backup`, inside -`retrobios-backup`, is ever deleted, and no link inside it is followed. On the +`retrobios-backup`, is ever deleted, and no link inside it is followed, a +Windows junction included. On the full screen, **More** on the plan, shown once the folder has a record or a backup, holds **Remove files**, **Backups**, **Put back a backup** and **Delete a backup**. Remove files lists every recorded file or one system,