fix: index each data directory once per run

scripts/common.py

@@ -635,21 +635,49 @@ def _resolve_in_data_dirs(names_to_try, data_dir_registry, verdict):
                 if status != "hash_mismatch":
                     return (candidate, status), mismatch
                 mismatch = mismatch or candidate
-        # The declared path may not be where the cache put it, so the whole
-        # tree is walked for the bare filename, case-insensitively.
+        # The declared path may not be where the cache put it, so the tree
+        # is searched for the bare filename, case-insensitively, in the order
+        # a walk meets it.
         basename_targets = {
             (n.rsplit("/", 1)[-1] if "/" in n else n).casefold()
             for n in names_to_try
         }
+        index = _data_dir_basenames(cache_dir)
+        found = sorted(
+            (order, path)
+            for target in basename_targets
+            for order, path in index.get(target, ())
+        )
+        for _order, candidate in found:
+            status = verdict(candidate)
+            if status != "hash_mismatch":
+                return (candidate, status), mismatch
+            mismatch = mismatch or candidate
+    return None, mismatch
+
+
+# Basename index per data-directory cache. Walking the whole tree again for
+# every unresolved name made resolution O(names x tree) on each pack build.
+_data_dir_index: dict[str, dict[str, list[tuple[int, str]]]] = {}
+
+
+def _data_dir_basenames(cache_dir: str) -> dict[str, list[tuple[int, str]]]:
+    """Casefolded basename -> (walk order, path) of every file in a cache."""
+    key = os.path.realpath(cache_dir)
+    if key not in _data_dir_index:
+        index: dict[str, list[tuple[int, str]]] = {}
+        order = 0
         for root, _dirs, fnames in os.walk(cache_dir):
             for fname in fnames:
-                if fname.casefold() in basename_targets:
-                    candidate = os.path.join(root, fname)
-                    status = verdict(candidate)
-                    if status != "hash_mismatch":
-                        return (candidate, status), mismatch
-                    mismatch = mismatch or candidate
-    return None, mismatch
+                index.setdefault(fname.casefold(), []).append((order, os.path.join(root, fname)))
+                order += 1
+        _data_dir_index[key] = index
+    return _data_dir_index[key]
+
+
+def forget_data_dir_index() -> None:
+    """Drop the basename indexes, after a data directory was refreshed."""
+    _data_dir_index.clear()
 
 
 def _resolve_agnostic(file_entry: dict, files_db: dict, has_strong_hash: bool):

scripts/refresh_data_dirs.py

@@ -491,6 +491,11 @@ def refresh_all(
             dry_run=dry_run,
             versions_path=versions_path,
         )
+    if any(results.values()):
+        # A cache the resolver indexed in this process has changed under it.
+        from common import forget_data_dir_index  # noqa: PLC0415
+
+        forget_data_dir_index()
     return results
 
 

tests/test_audit_regressions.py

@@ -2811,5 +2811,40 @@ class PipelineStepsSpeakAsTheyGo(unittest.TestCase):
         self.assertLess(text.index("two"), text.index("three"))
         self.assertLess(text.index("three"), text.index("--- probe: OK"))
 
+
+class ADataDirectoryIsWalkedOnce(unittest.TestCase):
+    """Each unresolved name walked every data-directory cache again: the
+    resolution cost names x tree on every pack build."""
+
+    def test_two_lookups_share_one_walk_and_keep_its_order(self):
+        import common  # noqa: PLC0415
+
+        with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as tmp:
+            cache = Path(tmp) / "cache"
+            (cache / "a").mkdir(parents=True)
+            (cache / "b").mkdir()
+            (cache / "a" / "Font.bin").write_bytes(b"1")
+            (cache / "b" / "font.bin").write_bytes(b"2")
+            (cache / "b" / "other.bin").write_bytes(b"3")
+            registry = {"x": {"local_cache": str(cache)}}
+            common.forget_data_dir_index()
+            walks = []
+            real_walk = os.walk
+
+            def counting_walk(top, *a, **k):
+                walks.append(top)
+                return real_walk(top, *a, **k)
+
+            with mock.patch.object(common.os, "walk", counting_walk):
+                first, _ = common._resolve_in_data_dirs(["font.bin"], registry,
+                                                        lambda _p: "name_exact")
+                second, _ = common._resolve_in_data_dirs(["other.bin"], registry,
+                                                         lambda _p: "name_exact")
+            common.forget_data_dir_index()
+        self.assertEqual(len(walks), 1)
+        expected = sorted(str(p) for p in (cache / "a" / "Font.bin", cache / "b" / "font.bin"))
+        self.assertIn(first[0], expected)
+        self.assertEqual(second[0], str(cache / "b" / "other.bin"))
+
 if __name__ == "__main__":
     unittest.main()