From 803f9dce96d4a953ce290f3d475d7e9ace7c0dbd Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Sun, 18 Oct 2026 07:12:59 +0200
Subject: [PATCH] fix: hash only files that share a size

identity.py read the whole 18 GB tree on every pipeline. A file whose
size no other file shares cannot have a duplicate.
---
 scripts/identity.py    | 17 +++++++++++++----
 tests/test_identity.py | 14 ++++++++++++++
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/scripts/identity.py b/scripts/identity.py
index 20aec481..381ed84d 100644
--- a/scripts/identity.py
+++ b/scripts/identity.py
@@ -93,13 +93,22 @@ def _sha1(path: Path) -> str:
 
 
 def scan(bios_dir: str = "bios") -> list[Duplicate]:
-    """Group the tree by content and keep the groups that cross makers."""
-    by_content: dict[str, list[Path]] = {}
+    """Group the tree by content and keep the groups that cross makers.
+
+    Only files that share a size can share a content, so a size held by one
+    file is never read: hashing the whole tree read 18 GB on every pipeline.
+    """
+    by_size: dict[int, list[Path]] = {}
     root = Path(bios_dir)
     for path in sorted(root.rglob("*")):
-        if not path.is_file():
+        if path.is_file():
+            by_size.setdefault(path.stat().st_size, []).append(path)
+    by_content: dict[str, list[Path]] = {}
+    for paths in by_size.values():
+        if len(paths) < 2:
             continue
-        by_content.setdefault(_sha1(path), []).append(path)
+        for path in paths:
+            by_content.setdefault(_sha1(path), []).append(path)
 
     found: list[Duplicate] = []
     for sha1, paths in by_content.items():
diff --git a/tests/test_identity.py b/tests/test_identity.py
index d2389db9..79e73b5d 100644
--- a/tests/test_identity.py
+++ b/tests/test_identity.py
@@ -92,6 +92,20 @@ class TestCrossVendorDuplicates(unittest.TestCase):
         self._write("Sega/Saturn/cartridge.bin", b"other")
         self.assertEqual(self._scan(), [])
 
+    def test_a_file_whose_size_no_other_shares_is_never_read(self):
+        """Hashing the whole tree read 18 GB on every pipeline run."""
+        from unittest import mock
+
+        self._write("Commodore/Amiga/kickstart.rom", b"kick")
+        self._write("Sega/Saturn/cartridge.bin", b"kick")
+        self._write("Sony/PlayStation/scph5501.bin", b"a longer dump")
+        self._write("Sega/Saturn/other.bin", b"same")
+        with mock.patch.object(identity, "_sha1", wraps=identity._sha1) as hashed:
+            found = self._scan()
+        read = sorted(Path(call.args[0]).name for call in hashed.call_args_list)
+        self.assertEqual(read, ["cartridge.bin", "kickstart.rom", "other.bin"])
+        self.assertEqual(len(found), 1)
+
     def test_report_line_names_every_identity(self):
         self._write("Commodore/Amiga/kickstart.rom", b"kick")
         self._write("Sega/Saturn/cartridge.bin", b"kick")