#!/usr/bin/env python3 """Deduplication, which permanently deletes files from the collection. dedup.py is the only script here that removes BIOS files, and it had no tests: the sole guard was remembering to pass --dry-run. These pin the rules it must never get wrong, above all the directories where two identical files are both load-bearing. """ from __future__ import annotations import contextlib import io import os import sys import tempfile import unittest from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT / "scripts")) import dedup # noqa: E402 class _Bios(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory() self.bios = Path(self._tmp.name) / "bios" self.bios.mkdir() def tearDown(self): self._tmp.cleanup() def write(self, relative: str, data: bytes = b"IDENTICAL PAYLOAD") -> Path: path = self.bios / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) return path def run_dedup(self, dry_run: bool = False) -> dict: with contextlib.redirect_stdout(io.StringIO()): return dedup.deduplicate(str(self.bios), dry_run=dry_run) def existing(self) -> set[str]: return { str(p.relative_to(self.bios)) for p in self.bios.rglob("*") if p.is_file() } class ProtectedDirectories(_Bios): """RPG Maker and ScummVM reference files by exact name and path. Collapsing two identical copies there breaks the reference that the removed path served, which is why CLAUDE.md marks them NODEDUP. """ def test_rpg_maker_copies_both_survive(self): self.write("RPG Maker/GameA/System/font.png") self.write("RPG Maker/GameB/System/font.png") self.run_dedup() self.assertEqual(len(self.existing()), 2, "a NODEDUP copy was removed") def test_scummvm_copies_both_survive(self): self.write("ScummVM/monkey/monkey.tbl") self.write("ScummVM/monkey2/monkey.tbl") self.run_dedup() self.assertEqual(len(self.existing()), 2) def test_a_protected_copy_does_not_drag_an_outside_copy_away(self): self.write("RPG Maker/GameA/System/shared.dat") self.write("Sony/PlayStation/shared.dat") self.run_dedup() self.assertIn("RPG Maker/GameA/System/shared.dat", self.existing()) class TrueDuplicates(_Bios): def test_the_same_name_in_two_directories_collapses_to_one(self): self.write("Arcade/MAME/naomi.zip") self.write("Sega/Dreamcast/naomi.zip") results = self.run_dedup() self.assertEqual(len(self.existing()), 1) removed = [r for entry in results.values() for r in entry["removed"]] self.assertEqual(len(removed), 1) def test_the_shorter_path_is_kept(self): self.write("Sony/PS/bios.bin") self.write("Sony/PS/deeply/nested/bios.bin") self.run_dedup() self.assertEqual(self.existing(), {"Sony/PS/bios.bin"}) def test_a_variant_loses_to_the_primary(self): self.write("Sony/PS/scph.bin") self.write("Sony/PS/.variants/scph.bin") self.run_dedup() self.assertEqual(self.existing(), {"Sony/PS/scph.bin"}) def test_different_content_under_one_name_is_never_touched(self): self.write("Sony/PS/a.bin", b"CONTENT ONE") self.write("Sega/Saturn/a.bin", b"CONTENT TWO") self.run_dedup() self.assertEqual(len(self.existing()), 2) def test_a_lone_file_is_left_alone(self): self.write("Sony/PS/only.bin") self.assertEqual(self.run_dedup(), {}) self.assertEqual(len(self.existing()), 1) class DifferentNamesSameContent(_Bios): def test_outside_mame_every_name_is_kept(self): """Two emulators may each look for their own spelling.""" self.write("Nintendo/N64/64DD_IPL_US.n64") self.write("Nintendo/N64/IPL_USA.n64") self.run_dedup() self.assertEqual(len(self.existing()), 2) def test_mame_zip_clones_collapse_and_are_recorded(self): self.write("Arcade/MAME/bbc_m87.zip") self.write("Arcade/MAME/bbc_24bbc.zip") self.run_dedup() survivors = self.existing() self.assertEqual(len(survivors), 1, f"expected one canonical, got {survivors}") def test_identically_named_mame_zips_keep_the_shortest_name(self): self.write("Arcade/MAME/aaa_long_name.zip") self.write("Arcade/MAME/bbb.zip") self.run_dedup() self.assertEqual(self.existing(), {"Arcade/MAME/bbb.zip"}) def test_a_non_zip_in_mame_is_not_treated_as_a_clone(self): self.write("Arcade/MAME/rom_one.bin") self.write("Arcade/MAME/rom_two.bin") self.run_dedup() self.assertEqual(len(self.existing()), 2) class DryRun(_Bios): def test_dry_run_deletes_nothing(self): self.write("Arcade/MAME/naomi.zip") self.write("Sega/Dreamcast/naomi.zip") before = self.existing() results = self.run_dedup(dry_run=True) self.assertEqual(self.existing(), before, "dry run removed a file") self.assertTrue( any(entry["removed"] for entry in results.values()), "dry run should still report what it would remove", ) def test_dry_run_reports_the_same_plan_the_real_run_executes(self): self.write("Arcade/MAME/naomi.zip") self.write("Sega/Dreamcast/naomi.zip") planned = { r for entry in self.run_dedup(dry_run=True).values() for r in entry["removed"] } done = { r for entry in self.run_dedup(dry_run=False).values() for r in entry["removed"] } self.assertEqual(planned, done) class EmptyVariantCleanup(_Bios): def test_an_emptied_variants_directory_is_removed(self): self.write("Sony/PS/scph.bin") self.write("Sony/PS/.variants/scph.bin") self.run_dedup() self.assertFalse((self.bios / "Sony" / "PS" / ".variants").exists()) def test_a_variants_directory_still_holding_a_file_stays(self): self.write("Sony/PS/scph.bin") self.write("Sony/PS/.variants/scph.bin") self.write("Sony/PS/.variants/other.bin", b"DIFFERENT") self.run_dedup() self.assertTrue((self.bios / "Sony" / "PS" / ".variants").is_dir()) class PathPriority(unittest.TestCase): def test_shorter_paths_sort_first(self): short = dedup.path_priority("bios/Sony/PS/a.bin") deep = dedup.path_priority("bios/Sony/PS/x/y/z/a.bin") self.assertLess(short, deep) def test_a_variant_sorts_after_a_primary(self): primary = dedup.path_priority("bios/Sony/PS/a.bin") variant = dedup.path_priority("bios/Sony/PS/.variants/a.bin") self.assertLess(primary, variant) def test_protected_directories_are_recognised(self): self.assertTrue(dedup._in_nodedup_dir("bios/RPG Maker/x/font.png")) self.assertTrue(dedup._in_nodedup_dir("bios/ScummVM/x/monkey.tbl")) self.assertFalse(dedup._in_nodedup_dir("bios/Sony/PlayStation/scph.bin")) if __name__ == "__main__": unittest.main()