chore: merge the installer backups

tests/test_install.py

@@ -4966,7 +4966,8 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
         installer = events[0]["installer"]
         self.assertEqual(events, self._head("check", installer) + [
             {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 1,
-             "pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False},
+             "pending": 3, "failed": [], "skipped": self.SKIPPED, "left_out": [], "kept": [],
+             "interrupted": False},
             {"event": "end", "code": 1},
         ])
         self.assertIn("2/4 present (1 verified, 1 wrong hash)", proc.stderr)
@@ -4976,6 +4977,9 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
         proc, events = self._run()
         self.assertEqual(proc.returncode, 0, proc.stderr)
         installer = events[0]["installer"]
+        kept = events[-2]["kept"]
+        self.assertEqual(len(kept), 1, kept)
+        self.assertEqual(Path(kept[0]["folder"]).parent, self.home / install.BACKUP_DIR)
         progress = [event for event in events if event["event"] == "progress"]
         for event in progress:
             self.assertEqual(list(event), ["event", *install.EVENT_KEYS["progress"]])
@@ -4989,7 +4993,9 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
             {"event": "download", "platform": "retroarch", "files": 3, "bytes": 245},
             *done,
             {"event": "summary", "mode": "install", "installed": 3, "up_to_date": 1,
-             "pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [], "interrupted": False},
+             "pending": 0, "failed": [], "skipped": self.SKIPPED, "left_out": [],
+             "kept": [{"folder": kept[0]["folder"], "files": 1, "bytes": 11}],
+             "interrupted": False},
             {"event": "end", "code": 0},
         ])
         self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])
@@ -5031,5 +5037,232 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
                 self.assertIn(f"`{key}`", row, f"{kind}.{key}")
 
 
+class AFileYouHadIsKept(unittest.TestCase):
+    """A BIOS file whose contents differ was replaced and lost: a personal
+    dump or another revision could not be brought back by deleting what the
+    installer wrote. It is moved beside the BIOS folder first, never inside,
+    where a core that finds its BIOS by content could take it."""
+
+    NEW = b"the collection's copy"
+    OLD = b"a dump of my own"
+
+    def setUp(self):
+        install.KEPT.reset()
+        self.addCleanup(install.KEPT.reset)
+        self.home = Path(_tempdir(self))
+        self.bios = self.home / "system"
+        self.bios.mkdir()
+
+    def _entry(self, dest: str) -> dict:
+        return {"dest": dest, "repo_path": f"bios/{dest}", "size": len(self.NEW),
+                "sha1": hashlib.sha1(self.NEW).hexdigest(),
+                "sha256": hashlib.sha256(self.NEW).hexdigest()}
+
+    def _download(self, dest: str):
+        with unittest.mock.patch.object(install.urllib.request, "urlopen",
+                                        lambda *a, **k: _FakeResponse(self.NEW)):
+            return install._download_one(self._entry(dest), self.bios)
+
+    def _run_folder(self) -> Path:
+        runs = list((self.home / install.BACKUP_DIR).iterdir())
+        self.assertEqual(len(runs), 1, runs)
+        return runs[0]
+
+    def test_a_differing_file_is_kept_with_its_relative_path(self):
+        (self.bios / "sub").mkdir()
+        (self.bios / "sub" / "x.bin").write_bytes(self.OLD)
+        self.assertEqual(self._download("sub/x.bin"), ("sub/x.bin", True, ""))
+        self.assertEqual((self.bios / "sub" / "x.bin").read_bytes(), self.NEW)
+        run = self._run_folder()
+        self.assertEqual((run / "system" / "sub" / "x.bin").read_bytes(), self.OLD)
+        self.assertEqual(install.KEPT.summary(),
+                         [{"folder": str(run), "files": 1, "bytes": len(self.OLD)}])
+        # Nothing new sits inside the BIOS folder, where a scan would read it.
+        self.assertEqual(sorted(p.name for p in self.bios.rglob("*")), ["sub", "x.bin"])
+
+    def test_an_identical_file_is_not_copied(self):
+        (self.bios / "x.bin").write_bytes(self.NEW)
+        self.assertEqual(self._download("x.bin"), ("x.bin", True, ""))
+        self.assertFalse((self.home / install.BACKUP_DIR).exists())
+        self.assertEqual(install.KEPT.summary(), [])
+
+    def test_a_missing_file_keeps_nothing(self):
+        self.assertEqual(self._download("x.bin"), ("x.bin", True, ""))
+        self.assertFalse((self.home / install.BACKUP_DIR).exists())
+
+    def test_a_second_replacement_keeps_the_first_file(self):
+        # Two lists in one run writing one path: what is kept is the user's.
+        (self.bios / "x.bin").write_bytes(self.OLD)
+        self._download("x.bin")
+        (self.bios / "x.bin").write_bytes(b"what this run wrote first")
+        self._download("x.bin")
+        self.assertEqual((self._run_folder() / "system" / "x.bin").read_bytes(), self.OLD)
+
+    def test_a_file_that_cannot_be_kept_stays(self):
+        (self.bios / "x.bin").write_bytes(self.OLD)
+
+        def refuse(*_args):
+            raise PermissionError(13, "Permission denied")
+
+        with unittest.mock.patch.object(install, "_move_aside", refuse):
+            _dest, ok, reason = self._download("x.bin")
+        self.assertFalse(ok)
+        self.assertIn(install.NOT_KEPT, reason)
+        self.assertEqual((self.bios / "x.bin").read_bytes(), self.OLD)
+        self.assertEqual(sorted(p.name for p in self.bios.iterdir()), ["x.bin"])
+
+    def test_a_symlinked_subfolder_keeps_working(self):
+        elsewhere = self.home / "shadPS4" / "sys_modules"
+        elsewhere.mkdir(parents=True)
+        (elsewhere / "lib.sprx").write_bytes(self.OLD)
+        (self.bios / "shadps4").mkdir()
+        (self.bios / "shadps4" / "sys_modules").symlink_to(elsewhere)
+        self.assertTrue(self._download("shadps4/sys_modules/lib.sprx")[1])
+        self.assertEqual((elsewhere / "lib.sprx").read_bytes(), self.NEW)
+        self.assertTrue((self.bios / "shadps4" / "sys_modules").is_symlink())
+        run = self._run_folder()
+        kept = run / "system" / "shadps4" / "sys_modules" / "lib.sprx"
+        self.assertEqual(kept.read_bytes(), self.OLD)
+        with contextlib.redirect_stdout(io.StringIO()):
+            self.assertEqual(install.restore_kept(run), 0)
+        self.assertEqual((elsewhere / "lib.sprx").read_bytes(), self.OLD)
+        self.assertTrue((self.bios / "shadps4" / "sys_modules").is_symlink())
+
+    def test_a_link_at_the_name_is_not_kept(self):
+        # The replace takes the link's place: the file it points to is untouched.
+        target = self.home / "mine.bin"
+        target.write_bytes(self.OLD)
+        (self.bios / "x.bin").symlink_to(target)
+        self.assertTrue(self._download("x.bin")[1])
+        self.assertEqual(target.read_bytes(), self.OLD)
+        self.assertFalse((self.home / install.BACKUP_DIR).exists())
+
+    @unittest.skipIf(hasattr(os, "geteuid") and os.geteuid() == 0, "root writes anywhere")
+    def test_without_room_beside_the_folder_it_goes_inside(self):
+        (self.bios / "x.bin").write_bytes(self.OLD)
+        self.home.chmod(0o555)
+        self.addCleanup(self.home.chmod, 0o755)
+        self.assertTrue(self._download("x.bin")[1])
+        runs = list((self.bios / install.BACKUP_DIR).iterdir())
+        self.assertEqual(len(runs), 1)
+        self.assertEqual((runs[0] / "x.bin").read_bytes(), self.OLD)
+        with contextlib.redirect_stdout(io.StringIO()):
+            self.assertEqual(install.restore_kept(runs[0]), 0)
+        self.assertEqual((self.bios / "x.bin").read_bytes(), self.OLD)
+
+    def test_a_data_tree_file_is_kept(self):
+        import zipfile
+
+        tree = self.bios / "dolphin-emu" / "Sys"
+        tree.mkdir(parents=True)
+        (tree / "a.ini").write_bytes(self.OLD)
+        archive = self.home / "src.zip"
+        with zipfile.ZipFile(archive, "w") as zf:
+            zf.writestr("Sys/a.ini", self.NEW)
+        entry = {"ref": "d", "dest": "dolphin-emu", "url": "https://example.invalid/d.zip",
+                 "type": "zip", "files": {"Sys/a.ini": {
+                     "sha1": hashlib.sha1(self.NEW).hexdigest(), "size": len(self.NEW)}}}
+
+        def fetch(url, directory):
+            copy = Path(directory) / "a.zip"
+            shutil.copy(archive, copy)
+            return copy
+
+        with unittest.mock.patch.object(install, "_fetch_archive", fetch):
+            counts = install.install_data_directory(entry, self.bios)
+        self.assertEqual(counts["installed"], 1)
+        self.assertEqual((tree / "a.ini").read_bytes(), self.NEW)
+        kept = self._run_folder() / "system" / "dolphin-emu" / "Sys" / "a.ini"
+        self.assertEqual(kept.read_bytes(), self.OLD)
+
+    def test_a_standalone_copy_keeps_the_emulators_file(self):
+        (self.bios / "ps2-0230a.bin").write_bytes(self.NEW)
+        pcsx2 = self.home / "PCSX2" / "bios"
+        pcsx2.mkdir(parents=True)
+        (pcsx2 / "ps2-0230a.bin").write_bytes(self.OLD)
+        manifest = {"standalone_copies": [
+            {"pattern": "ps2-*.bin", "emulator": "pcsx2", "targets": {"linux": [str(pcsx2)]}}]}
+        self.assertEqual(install.do_standalone_copies(manifest, self.bios, "linux"), (1, 0, 0))
+        self.assertEqual((pcsx2 / "ps2-0230a.bin").read_bytes(), self.NEW)
+        runs = list((self.home / "PCSX2" / install.BACKUP_DIR).iterdir())
+        self.assertEqual((runs[0] / "bios" / "ps2-0230a.bin").read_bytes(), self.OLD)
+        # Copied again, the file is the same: nothing more is kept.
+        self.assertEqual(install.do_standalone_copies(manifest, self.bios, "linux"), (1, 0, 0))
+        self.assertEqual(install.KEPT.summary()[0]["files"], 1)
+
+    def test_a_kept_file_inside_the_tree_is_no_copy_source(self):
+        inside = self.bios / install.BACKUP_DIR / "20261022-153000"
+        inside.mkdir(parents=True)
+        (inside / "ps2-old.bin").write_bytes(self.OLD)
+        pcsx2 = self.home / "PCSX2" / "bios"
+        pcsx2.mkdir(parents=True)
+        manifest = {"standalone_copies": [
+            {"pattern": "ps2-*.bin", "emulator": "pcsx2", "targets": {"linux": [str(pcsx2)]}}]}
+        self.assertEqual(install.do_standalone_copies(manifest, self.bios, "linux"), (0, 0, 0))
+
+    def test_restore_refuses_a_folder_that_is_no_backup(self):
+        with contextlib.redirect_stderr(io.StringIO()) as err:
+            self.assertEqual(install.restore_kept(self.bios), 2)
+        self.assertIn(install.BACKUP_DIR, err.getvalue())
+
+
+class KeptFilesEndToEnd(unittest.TestCase, _SyntheticTree):
+    """The run says what it kept and where, --dry-run what it would keep
+    and writes nothing, and --restore puts the original bytes back."""
+
+    def setUp(self):
+        self.env = self.serve_tree(self)
+
+    def test_dry_run_says_what_would_be_kept_and_writes_nothing(self):
+        proc = _install_run(self.env, "--platform", "retroarch", "--dest", str(self.bios),
+                            "--dry-run", "--no-input")
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        self.assertIn("Would keep 1 file you have (11 B)", proc.stdout)
+        self.assertIn(str(self.home / install.BACKUP_DIR), proc.stdout)
+        self.assertFalse((self.home / install.BACKUP_DIR).exists())
+        self.assertEqual((self.bios / "b.bin").read_bytes(), b"other bytes")
+        events = _events(_install_run(self.env, "--platform", "retroarch", "--dest",
+                                      str(self.bios), "--dry-run", "--events"))
+        kept = events[-2]["kept"]
+        self.assertEqual([(item["files"], item["bytes"]) for item in kept], [(1, 11)])
+        self.assertFalse((self.home / install.BACKUP_DIR).exists())
+
+    def test_install_then_restore(self):
+        proc = _install_run(self.env, "--platform", "retroarch", "--dest", str(self.bios),
+                            "--no-input", "--jobs", "1")
+        self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
+        self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])
+        runs = list((self.home / install.BACKUP_DIR).iterdir())
+        self.assertEqual(len(runs), 1)
+        self.assertIn("Kept 1 file you had (11 B), which the collection's copies "
+                      f"replaced, in {runs[0]}", proc.stdout)
+        self.assertIn(f"--restore {runs[0]}", proc.stdout)
+        # a.bin held the expected bytes: it is not among the kept files.
+        self.assertEqual([p.name for p in (runs[0] / "bios").iterdir()], ["b.bin"])
+
+        refused = _install_run(self.env, "--restore", str(runs[0]), "--platform", "retroarch")
+        self.assertEqual(refused.returncode, 2)
+        self.assertIn("--restore", refused.stderr)
+
+        looked = _install_run(self.env, "--restore", str(runs[0]), "--dry-run")
+        self.assertEqual(looked.returncode, 0, looked.stderr)
+        self.assertIn(f"would put back {self.bios / 'b.bin'}", looked.stdout)
+        self.assertEqual((self.bios / "b.bin").read_bytes(), self.FILES["bios/A/b.bin"])
+
+        back = _install_run(self.env, "--restore", str(runs[0]))
+        self.assertEqual(back.returncode, 0, back.stdout + back.stderr)
+        self.assertEqual((self.bios / "b.bin").read_bytes(), b"other bytes")
+        self.assertEqual((self.bios / "a.bin").read_bytes(), self.FILES["bios/A/a.bin"])
+        self.assertIn("Put back 1 file", back.stdout)
+        # The backup stays until the user deletes it.
+        self.assertTrue((runs[0] / "bios" / "b.bin").is_file())
+
+        # A second install keeps the restored file in a new folder.
+        again = _install_run(self.env, "--platform", "retroarch", "--dest", str(self.bios),
+                             "--no-input", "--jobs", "1")
+        self.assertEqual(again.returncode, 0, again.stderr)
+        self.assertEqual(len(list((self.home / install.BACKUP_DIR).iterdir())), 2)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_port.py

@@ -141,7 +141,9 @@ class EventRecords(unittest.TestCase):
             json.dumps({"event": "summary", "mode": "install", "installed": 0,
                         "up_to_date": 4, "pending": 1,
                         "failed": [{"dest": "x.bin", "reason": "HTTP 404"}],
-                        "skipped": [], "left_out": [], "interrupted": False}),
+                        "skipped": [], "left_out": [],
+                        "kept": [{"folder": "/b/retrobios-backup/1", "files": 1,
+                                  "bytes": 2048}], "interrupted": False}),
             json.dumps({"event": "end", "code": 2}),
         ]
         out, summary = io.StringIO(), io.StringIO()
@@ -149,7 +151,8 @@ class EventRecords(unittest.TestCase):
         self.assertEqual(out.getvalue().splitlines(), [
             "download\t1\t2 KB", "file\t1\t1\tfailed\tx.bin\t0 B\t2 KB"])
         self.assertEqual(summary.getvalue().splitlines(), [
-            "summary\t0\t4\t1\t0", "failed\tx.bin\tHTTP 404", "end\t2"])
+            "summary\t0\t4\t1\t0", "failed\tx.bin\tHTTP 404",
+            "kept\t/b/retrobios-backup/1\t1\t2 KB", "end\t2"])
 
     def test_a_system_id_that_is_no_slug_is_dropped(self):
         # The id becomes a bash array subscript in the launcher, where a
@@ -228,7 +231,7 @@ if check:
          present=3 if done else 1, missing=0 if done else 1, wrong_hash=0 if done else 1,
          bytes_needed=0 if done else 300, data_missing=0, not_installable=1)
     emit("summary", mode="check", installed=0, up_to_date=3 if done else 1,
-         pending=0 if done else 2, failed=[], skipped=[], left_out=[],
+         pending=0 if done else 2, failed=[], skipped=[], left_out=[], kept=[],
          interrupted=False)
     end(0 if done else 1)
 emit("download", platform="retroarch", files=2, bytes=300)
@@ -238,7 +241,8 @@ for n, (dest, size) in enumerate((("a.bin", 100), ("b.bin", 200)), 1):
 (state / "installed").write_text("")
 emit("summary", mode="install", installed=2, up_to_date=1, pending=0, failed=[],
      skipped=[{"reason": "not_found", "label": "not found", "files": 1, "required": 1}],
-     left_out=[], interrupted=False)
+     left_out=[], kept=[{"folder": "/roms/retrobios-backup/20261022-153000", "files": 1,
+                         "bytes": 100}], interrupted=False)
 end(0)
 '''
 
@@ -371,7 +375,8 @@ class TheScreens(unittest.TestCase):
         shown = self.run_port("missing", ["update", "back", "quit"])
         self.assertEqual(self.lists(shown), [
             ("menu", ["status", "update", "choose", "check", "quit"]),
-            ("summary", ["info:result", "back", "quit", "info:skipped:not found"]),
+            ("summary", ["info:result", "back", "quit", "info:kept:1",
+                         "info:skipped:not found"]),
             ("menu", ["status", "check", "quit"]),
         ])
         menu = shown[[s[0] for s in shown].index("list")][2]
@@ -385,6 +390,8 @@ class TheScreens(unittest.TestCase):
         ])
         summary = [s for s in shown if s[:2] == ("list", "summary")][0][2]
         self.assertEqual(summary["info:result"][0], "Installed: 2 files")
+        self.assertEqual(summary["info:kept:1"][0], "Kept 1 file you had (100 B)")
+        self.assertIn("/roms/retrobios-backup/20261022-153000", summary["info:kept:1"][1])
         self.assertEqual(self.calls(), [
             "--check --events",
             "--events --platform retroarch",