From 78e172fd5f9823a6c1daef1f2177396b1bcdf97e Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Wed, 21 Oct 2026 10:08:25 +0200
Subject: [PATCH] fix: index archive members once per profile

archive_members scanned the whole file list on every call, and callers ask once per entry of the same profile: 38 million entries walked in one verify --all over mame.yml. Lists of 64 entries or more are now indexed by archive once (held so the id is not reused, indexed again when the length changes), members still read from the entries at each call. archive_members drops from 2.30 s to 0.14 s tottime under the profiler. verify --all (text and --json), slots --json and the retroarch manifest are byte-identical.
---
 scripts/common.py            | 29 +++++++++++++++++++----
 tests/test_archive_choice.py | 45 ++++++++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 5 deletions(-)

diff --git a/scripts/common.py b/scripts/common.py
index 35b53465..249b65a2 100644
--- a/scripts/common.py
+++ b/scripts/common.py
@@ -1018,13 +1018,32 @@ def _rank_tail_candidates(
     )
 
 
+# id(files) -> (files, len(files), {archive: entries with a crc32}). Callers
+# ask once per entry of the same profile: scanning the list each time walked
+# 38 million entries in one verify --all over mame.yml. The list is held so
+# its id is not reused, a list that changed length is indexed again, and the
+# members are read from the entries at each call. Short lists are scanned
+# directly: a fresh default [] per call would only fill the cache.
+_ARCHIVE_INDEX_MIN = 64
+_archive_index: dict[int, tuple[list[dict], int, dict[str | None, list[dict]]]] = {}
+
+
+def _member(f: dict) -> dict:
+    return {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
+
+
 def archive_members(archive: str, files: list[dict]) -> list[dict]:
     """The members a profile reads inside *archive*, with what it requires."""
-    return [
-        {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
-        for f in files
-        if f.get("archive") == archive and f.get("crc32")
-    ]
+    if len(files) < _ARCHIVE_INDEX_MIN:
+        return [_member(f) for f in files if f.get("archive") == archive and f.get("crc32")]
+    cached = _archive_index.get(id(files))
+    if cached is None or cached[0] is not files or cached[1] != len(files):
+        index: dict[str | None, list[dict]] = {}
+        for f in files:
+            if f.get("crc32"):
+                index.setdefault(f.get("archive"), []).append(f)
+        cached = _archive_index[id(files)] = (files, len(files), index)
+    return [_member(f) for f in cached[2].get(archive, ())]
 
 
 def archive_entry(archive: str, files: list[dict]) -> dict:
diff --git a/tests/test_archive_choice.py b/tests/test_archive_choice.py
index 046742d3..039d11d2 100644
--- a/tests/test_archive_choice.py
+++ b/tests/test_archive_choice.py
@@ -409,5 +409,50 @@ class ALooseEntryNamingItsArchive(unittest.TestCase):
         self.assertEqual(report["total_files"], 1)
 
 
+class ArchiveMembersIndexOnce(unittest.TestCase):
+    """with_archive_members asks once per entry of a profile: scanning the
+    whole list for each walked 38 million entries in one verify --all."""
+
+    class CountingList(list):
+        walks = 0
+
+        def __iter__(self):
+            type(self).walks += 1
+            return super().__iter__()
+
+    def entries(self) -> list[dict]:
+        files = self.CountingList()
+        self.CountingList.walks = 0
+        for n in range(300):
+            files.append({"name": f"m{n}.rom", "archive": f"set{n % 30}.zip",
+                          "crc32": f"{n:08X}", "required": n % 3 != 0})
+        files.append({"name": "loose.bin", "crc32": "deadbeef"})
+        files.append({"name": "nocrc.rom", "archive": "set0.zip"})
+        return files
+
+    @staticmethod
+    def scanned(archive, files) -> list[dict]:
+        return [{"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
+                for f in list.__iter__(files) if f.get("archive") == archive and f.get("crc32")]
+
+    def test_a_profile_list_is_walked_once_and_answers_as_a_scan(self):
+        from common import archive_members
+
+        files = self.entries()
+        for archive in [f"set{n}.zip" for n in range(30)] + ["absent.zip", "", None]:
+            with self.subTest(archive=archive):
+                self.assertEqual(archive_members(archive, files), self.scanned(archive, files))
+        self.assertEqual(self.CountingList.walks, 1)
+
+    def test_a_list_that_grew_is_indexed_again(self):
+        from common import archive_members
+
+        files = self.entries()
+        archive_members("set1.zip", files)
+        files.append({"name": "late.rom", "archive": "set1.zip", "crc32": "ABCDEF01"})
+        self.assertEqual(archive_members("set1.zip", files), self.scanned("set1.zip", files))
+        self.assertIn({"crc32": "abcdef01", "required": True}, archive_members("set1.zip", files))
+
+
 if __name__ == "__main__":
     unittest.main()