From 11c836cdeedef8916f27c337d3f10376a0ce5b04 Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Wed, 21 Oct 2026 10:00:53 +0200
Subject: [PATCH] fix: hash only same-size files in dedup scan

Equal files have equal sizes, and the scan only uses the sha1. Files alone at their size are no longer read and the rest get one digest instead of five. Same 278 groups (645 paths), same dry-run output, 36.7 s to 1.9 s.
---
 scripts/dedup.py    | 20 +++++++++++++++-----
 tests/test_dedup.py | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/scripts/dedup.py b/scripts/dedup.py
index 7c7f9a2f..6c38b7cf 100644
--- a/scripts/dedup.py
+++ b/scripts/dedup.py
@@ -97,17 +97,27 @@ def _is_mame_dir(path: str) -> bool:
 
 
 def scan_duplicates(bios_dir: str) -> dict[str, list[str]]:
-    """Find all files grouped by SHA1, excluding no-dedup directories."""
-    sha1_to_paths: dict[str, list[str]] = defaultdict(list)
+    """Files grouped by SHA1, excluding no-dedup directories.
 
-    for root, dirs, files in os.walk(bios_dir):
+    Two equal files have the same size, so only files that share a size are
+    read, and for the sha1 alone: hashing the whole tree with every digest
+    took 33 s for a scan that reads a quarter of it. A file alone at its
+    size is in no group of two and is left out.
+    """
+    by_size: dict[int, list[str]] = defaultdict(list)
+    for root, _dirs, files in os.walk(bios_dir):
         for name in files:
             path = os.path.join(root, name)
             if _in_nodedup_dir(path):
                 continue
-            sha1 = compute_hashes(path)["sha1"]
-            sha1_to_paths[sha1].append(path)
+            by_size[os.path.getsize(path)].append(path)
 
+    sha1_to_paths: dict[str, list[str]] = defaultdict(list)
+    for paths in by_size.values():
+        if len(paths) < 2:
+            continue
+        for path in paths:
+            sha1_to_paths[compute_hashes(path, frozenset({"sha1"}))["sha1"]].append(path)
     return sha1_to_paths
 
 
diff --git a/tests/test_dedup.py b/tests/test_dedup.py
index 5bab6a64..02a1207d 100644
--- a/tests/test_dedup.py
+++ b/tests/test_dedup.py
@@ -309,5 +309,37 @@ class TheNoDedupListIsPinned(unittest.TestCase):
         self.assertEqual(expected - set(NODEDUP_DIRS), set())
 
 
+class OnlyFilesSharingASizeAreHashed(_Bios):
+    """Equal files have equal sizes. Hashing all 9 442 files with five
+    digests took 33 s for 278 groups that a sha1 of the 4 543 files sharing
+    a size finds the same."""
+
+    def test_a_file_alone_at_its_size_is_not_read(self):
+        from unittest import mock
+
+        self.write("A/one.bin")
+        self.write("B/one.bin")
+        self.write("C/other.bin", b"a payload of its own length")
+        self.write("D/same_size.bin", b"IDENTICAL PAYLOAX")
+        calls: list[tuple[str, frozenset | None]] = []
+        real = dedup.compute_hashes
+
+        def spy(path, algorithms=None, cache=True):
+            calls.append((os.path.relpath(path, self.bios), algorithms))
+            return real(path, algorithms, cache)
+
+        with mock.patch.object(dedup, "compute_hashes", spy):
+            groups = dedup.scan_duplicates(str(self.bios))
+        self.assertEqual(
+            sorted(path for path, _algos in calls),
+            sorted(os.path.join(*p.split("/")) for p in ("A/one.bin", "B/one.bin", "D/same_size.bin")),
+        )
+        self.assertTrue(all(algos == frozenset({"sha1"}) for _path, algos in calls))
+        multi = [paths for paths in groups.values() if len(paths) > 1]
+        self.assertEqual(len(multi), 1)
+        self.assertEqual(sorted(os.path.relpath(p, self.bios) for p in multi[0]),
+                         sorted(os.path.join(*p.split("/")) for p in ("A/one.bin", "B/one.bin")))
+
+
 if __name__ == "__main__":
     unittest.main()