fix: hash only same-size files in dedup scan

Equal files have equal sizes, and the scan only uses the sha1. Files alone at their size are no longer read and the rest get one digest instead of five. Same 278 groups (645 paths), same dry-run output, 36.7 s to 1.9 s.

scripts/dedup.py

@@ -97,17 +97,27 @@ def _is_mame_dir(path: str) -> bool:
 
 
 def scan_duplicates(bios_dir: str) -> dict[str, list[str]]:
-    """Find all files grouped by SHA1, excluding no-dedup directories."""
-    sha1_to_paths: dict[str, list[str]] = defaultdict(list)
+    """Files grouped by SHA1, excluding no-dedup directories.
 
-    for root, dirs, files in os.walk(bios_dir):
+    Two equal files have the same size, so only files that share a size are
+    read, and for the sha1 alone: hashing the whole tree with every digest
+    took 33 s for a scan that reads a quarter of it. A file alone at its
+    size is in no group of two and is left out.
+    """
+    by_size: dict[int, list[str]] = defaultdict(list)
+    for root, _dirs, files in os.walk(bios_dir):
         for name in files:
             path = os.path.join(root, name)
             if _in_nodedup_dir(path):
                 continue
-            sha1 = compute_hashes(path)["sha1"]
-            sha1_to_paths[sha1].append(path)
+            by_size[os.path.getsize(path)].append(path)
 
+    sha1_to_paths: dict[str, list[str]] = defaultdict(list)
+    for paths in by_size.values():
+        if len(paths) < 2:
+            continue
+        for path in paths:
+            sha1_to_paths[compute_hashes(path, frozenset({"sha1"}))["sha1"]].append(path)
     return sha1_to_paths
 
 

tests/test_dedup.py

@@ -309,5 +309,37 @@ class TheNoDedupListIsPinned(unittest.TestCase):
         self.assertEqual(expected - set(NODEDUP_DIRS), set())
 
 
+class OnlyFilesSharingASizeAreHashed(_Bios):
+    """Equal files have equal sizes. Hashing all 9 442 files with five
+    digests took 33 s for 278 groups that a sha1 of the 4 543 files sharing
+    a size finds the same."""
+
+    def test_a_file_alone_at_its_size_is_not_read(self):
+        from unittest import mock
+
+        self.write("A/one.bin")
+        self.write("B/one.bin")
+        self.write("C/other.bin", b"a payload of its own length")
+        self.write("D/same_size.bin", b"IDENTICAL PAYLOAX")
+        calls: list[tuple[str, frozenset | None]] = []
+        real = dedup.compute_hashes
+
+        def spy(path, algorithms=None, cache=True):
+            calls.append((os.path.relpath(path, self.bios), algorithms))
+            return real(path, algorithms, cache)
+
+        with mock.patch.object(dedup, "compute_hashes", spy):
+            groups = dedup.scan_duplicates(str(self.bios))
+        self.assertEqual(
+            sorted(path for path, _algos in calls),
+            sorted(os.path.join(*p.split("/")) for p in ("A/one.bin", "B/one.bin", "D/same_size.bin")),
+        )
+        self.assertTrue(all(algos == frozenset({"sha1"}) for _path, algos in calls))
+        multi = [paths for paths in groups.values() if len(paths) > 1]
+        self.assertEqual(len(multi), 1)
+        self.assertEqual(sorted(os.path.relpath(p, self.bios) for p in multi[0]),
+                         sorted(os.path.join(*p.split("/")) for p in ("A/one.bin", "B/one.bin")))
+
+
 if __name__ == "__main__":
     unittest.main()