refactor: index profile entries by name and hash

_matching walked the reader's whole profile for each platform entry, 2,135 scans of MAME in one verify --all. Each profile is now indexed once per kind by name and by accepted hash. Over every registered platform, 10,524 lookups give the same answers as the scan, in 0.13 s instead of 2.42 s.

scripts/exporter/candidates.py

@@ -375,6 +375,7 @@ class EntryCategories:
         # entry with a proof walks its reader's whole profile, and a MAME
         # profile's archive members were hashed again for each.
         self._accepted: dict[int, set[str]] = {}
+        self._entry_index: dict[tuple[str, str], tuple[dict, dict]] = {}
         # One index per core and per kind, so a proof names its reader.
         self._readers: list[tuple[str, str, ReadEvidence]] = []
         for key in sorted(examined):
@@ -418,6 +419,24 @@ class EntryCategories:
             accepted = self._accepted[id(profile_entry)] = _entry_hashes(profile_entry)
         return accepted
 
+    def _index_of(self, key: str, kind: str) -> tuple[dict, dict]:
+        """A profile's entries of one kind by casefolded name and by accepted
+        hash, each list in file order. Built once per profile and kind: a
+        scan per platform entry cost 2 s of verify --all on MAME alone."""
+        index = self._entry_index.get((key, kind))
+        if index is None:
+            by_name: dict[str, list[tuple[int, dict]]] = {}
+            by_hash: dict[str, list[tuple[int, dict]]] = {}
+            for position, f in enumerate(self._profiles[key].get("files") or []):
+                if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA):
+                    continue
+                for n in {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n}:
+                    by_name.setdefault(n, []).append((position, f))
+                for value in self._accepted_by(f):
+                    by_hash.setdefault(value, []).append((position, f))
+            index = self._entry_index[(key, kind)] = (by_name, by_hash)
+        return index
+
     def _matching(self, key: str, kind: str, entry: dict) -> list[dict]:
         """The profile entries of `key` that read the platform entry.
 
@@ -434,19 +453,28 @@ class EntryCategories:
             if n
         }
         declared = _declared_hashes(entry)
-        by_hash: list[dict] = []
-        by_name: list[dict] = []
-        for f in self._profiles[key].get("files") or []:
-            if not isinstance(f, dict) or (f.get("category") == GAME_DATA) != (kind == GAME_DATA):
-                continue
-            cited = {k: f[k] for k in ("name", "source_ref") if f.get(k)}
-            if declared and _accepts(self._accepted_by(f), declared):
-                by_hash.append(cited)
-                continue
-            own = {str(n).casefold() for n in (f.get("name"), *(f.get("aliases") or [])) if n}
-            if own & names and not (declared and _contradicts(f, declared)):
-                by_name.append({**cited, "by": "name"})
-        return by_hash or by_name
+        by_name_index, by_hash_index = self._index_of(key, kind)
+
+        def cited(f: dict) -> dict:
+            return {k: f[k] for k in ("name", "source_ref") if f.get(k)}
+
+        found: dict[int, dict] = {}
+        for values in declared.values():
+            for value in values:
+                for position, f in by_hash_index.get(value, ()):
+                    found[position] = f
+                if _CUT_MD5 <= len(value) < _FULL_MD5:
+                    for accepted, hits in by_hash_index.items():
+                        if accepted.startswith(value):
+                            found.update(hits)
+        if found:
+            return [cited(found[p]) for p in sorted(found)]
+        named = {position: f for n in names for position, f in by_name_index.get(n, ())}
+        return [
+            {**cited(named[p]), "by": "name"}
+            for p in sorted(named)
+            if not (declared and _contradicts(named[p], declared))
+        ]
 
     def explain(self, entry: dict, destination: str, system: str) -> dict:
         """The category of an entry and what establishes it."""

tests/test_entry_categories.py

@@ -220,5 +220,49 @@ class IndexedProof(unittest.TestCase):
                 self.assertEqual(index.proofs(entry, destination), expected)
 
 
+class AProfileIsIndexedOncePerKind(unittest.TestCase):
+    """_matching walked the reader's whole profile for each platform entry:
+    2,135 scans of the MAME profile took 2 s of verify --all."""
+
+    FILES = [
+        {"name": "zero.bin", "md5": "0" * 32, "source_ref": "a.c:1"},
+        {"name": "one.bin", "md5": "1" * 32, "source_ref": "a.c:2"},
+        {"name": "one.bin", "md5": "2" * 32, "source_ref": "a.c:3"},
+        {"name": "two.bin", "aliases": ["TWO.ROM"], "source_ref": "a.c:4"},
+        {"name": "data.pak", "category": "game_data", "source_ref": "a.c:5"},
+    ]
+
+    def setUp(self):
+        profiles = {"core": {"emulator": "Core", "type": "libretro", "cores": ["core"],
+                             "systems": ["s"], "files": self.FILES}}
+        self.cats = EntryCategories(_config(["core"], []), profiles)
+
+    def test_the_index_answers_as_the_scan_did(self):
+        cases = [
+            ({"name": "one.bin", "md5": "2" * 32}, BIOS, [{"name": "one.bin", "source_ref": "a.c:3"}]),
+            # A cut md5 (Batocera's 29 characters) matches by prefix.
+            ({"name": "x", "md5": "1" * 29}, BIOS, [{"name": "one.bin", "source_ref": "a.c:2"}]),
+            ({"name": "one.bin"}, BIOS, [
+                {"name": "one.bin", "source_ref": "a.c:2", "by": "name"},
+                {"name": "one.bin", "source_ref": "a.c:3", "by": "name"},
+            ]),
+            ({"name": "one.bin", "md5": "9" * 32}, BIOS, []),
+            ({"name": "two.rom"}, BIOS, [{"name": "two.bin", "source_ref": "a.c:4", "by": "name"}]),
+            ({"name": "data.pak"}, BIOS, []),
+            ({"name": "data.pak"}, GAME_DATA, [{"name": "data.pak", "source_ref": "a.c:5", "by": "name"}]),
+        ]
+        for entry, kind, expected in cases:
+            with self.subTest(entry=entry, kind=kind):
+                self.assertEqual(self.cats._matching("core", kind, entry), expected)
+
+    def test_the_profile_is_walked_once_per_kind(self):
+        from unittest import mock
+
+        with mock.patch.object(self.cats, "_accepted_by", wraps=self.cats._accepted_by) as hashed:
+            for _ in range(50):
+                self.cats._matching("core", BIOS, {"name": "zero.bin", "md5": "0" * 32})
+        self.assertEqual(hashed.call_count, 4)
+
+
 if __name__ == "__main__":
     unittest.main()