fix: index archive members once per profile

archive_members scanned the whole file list on every call, and callers ask once per entry of the same profile: 38 million entries walked in one verify --all over mame.yml. Lists of 64 entries or more are now indexed by archive once (held so the id is not reused, indexed again when the length changes), members still read from the entries at each call. archive_members drops from 2.30 s to 0.14 s tottime under the profiler. verify --all (text and --json), slots --json and the retroarch manifest are byte-identical.

scripts/common.py

@@ -1018,13 +1018,32 @@ def _rank_tail_candidates(
     )
 
 
+# id(files) -> (files, len(files), {archive: entries with a crc32}). Callers
+# ask once per entry of the same profile: scanning the list each time walked
+# 38 million entries in one verify --all over mame.yml. The list is held so
+# its id is not reused, a list that changed length is indexed again, and the
+# members are read from the entries at each call. Short lists are scanned
+# directly: a fresh default [] per call would only fill the cache.
+_ARCHIVE_INDEX_MIN = 64
+_archive_index: dict[int, tuple[list[dict], int, dict[str | None, list[dict]]]] = {}
+
+
+def _member(f: dict) -> dict:
+    return {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
+
+
 def archive_members(archive: str, files: list[dict]) -> list[dict]:
     """The members a profile reads inside *archive*, with what it requires."""
-    return [
-        {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
-        for f in files
-        if f.get("archive") == archive and f.get("crc32")
-    ]
+    if len(files) < _ARCHIVE_INDEX_MIN:
+        return [_member(f) for f in files if f.get("archive") == archive and f.get("crc32")]
+    cached = _archive_index.get(id(files))
+    if cached is None or cached[0] is not files or cached[1] != len(files):
+        index: dict[str | None, list[dict]] = {}
+        for f in files:
+            if f.get("crc32"):
+                index.setdefault(f.get("archive"), []).append(f)
+        cached = _archive_index[id(files)] = (files, len(files), index)
+    return [_member(f) for f in cached[2].get(archive, ())]
 
 
 def archive_entry(archive: str, files: list[dict]) -> dict:

tests/test_archive_choice.py

@@ -409,5 +409,50 @@ class ALooseEntryNamingItsArchive(unittest.TestCase):
         self.assertEqual(report["total_files"], 1)
 
 
+class ArchiveMembersIndexOnce(unittest.TestCase):
+    """with_archive_members asks once per entry of a profile: scanning the
+    whole list for each walked 38 million entries in one verify --all."""
+
+    class CountingList(list):
+        walks = 0
+
+        def __iter__(self):
+            type(self).walks += 1
+            return super().__iter__()
+
+    def entries(self) -> list[dict]:
+        files = self.CountingList()
+        self.CountingList.walks = 0
+        for n in range(300):
+            files.append({"name": f"m{n}.rom", "archive": f"set{n % 30}.zip",
+                          "crc32": f"{n:08X}", "required": n % 3 != 0})
+        files.append({"name": "loose.bin", "crc32": "deadbeef"})
+        files.append({"name": "nocrc.rom", "archive": "set0.zip"})
+        return files
+
+    @staticmethod
+    def scanned(archive, files) -> list[dict]:
+        return [{"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
+                for f in list.__iter__(files) if f.get("archive") == archive and f.get("crc32")]
+
+    def test_a_profile_list_is_walked_once_and_answers_as_a_scan(self):
+        from common import archive_members
+
+        files = self.entries()
+        for archive in [f"set{n}.zip" for n in range(30)] + ["absent.zip", "", None]:
+            with self.subTest(archive=archive):
+                self.assertEqual(archive_members(archive, files), self.scanned(archive, files))
+        self.assertEqual(self.CountingList.walks, 1)
+
+    def test_a_list_that_grew_is_indexed_again(self):
+        from common import archive_members
+
+        files = self.entries()
+        archive_members("set1.zip", files)
+        files.append({"name": "late.rom", "archive": "set1.zip", "crc32": "ABCDEF01"})
+        self.assertEqual(archive_members("set1.zip", files), self.scanned("set1.zip", files))
+        self.assertIn({"crc32": "abcdef01", "required": True}, archive_members("set1.zip", files))
+
+
 if __name__ == "__main__":
     unittest.main()