diff --git a/scripts/packextras.py b/scripts/packextras.py index 23d13e4b1..900c21658 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -253,11 +253,27 @@ class ExtraResolver: self.validation_index = validation_index self.validation_owners = validation_owners self.offline = offline + # Each profile's entries by slot, built on first use: every extra + # scanned its whole profile, 33 million entries in one verify --all. + self._slots: dict[str, dict[tuple[str, str], list[dict]]] = {} + + def alternatives(self, fe: dict) -> list[dict]: + """The versions the extra's own profile accepts at its slot, as + validation.extra_alternatives finds them.""" + from validation import slot_index + + key = fe.get("source_profile") or "" + index = self._slots.get(key) + if index is None: + profile = (self.profiles or {}).get(key, {}) + index = self._slots[key] = slot_index(profile.get("files") or []) + name = fe.get("source_name") or fe.get("name", "") + return index.get((name, fe.get("source_path") or name), []) def resolve(self, fe: dict, dest: str) -> tuple[str | None, str, str]: """(local copy or None, resolver status, the core's disagreement).""" from packresolve import resolve_file - from validation import core_size_refusal, extra_alternatives, validated_choice + from validation import core_size_refusal, validated_choice local_path, status = resolve_file( fe, self.db, self.bios_dir, self.zip_contents, @@ -272,9 +288,7 @@ class ExtraResolver: ) # The platform never reads the bytes, the core does: a copy it turns # away by its own size check is not its file, on any platform. - if core_size_refusal( - fe, local_path, extra_alternatives(fe, self.profiles) - ): + if core_size_refusal(fe, local_path, self.alternatives(fe)): return None, "not_found", "" return local_path, status, disagreement diff --git a/scripts/validation.py b/scripts/validation.py index 7a8e9dad2..300e36d05 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -1004,6 +1004,17 @@ def same_slot_entries(files: list, name: str, path: str) -> list[dict]: ] +def slot_index(files: list) -> dict[tuple[str, str], list[dict]]: + """A profile's entries keyed by (name, path), as same_slot_entries + matches them: one pass, then one lookup per extra.""" + index: dict[tuple[str, str], list[dict]] = {} + for entry in files or []: + if isinstance(entry, dict): + name = entry.get("name") + index.setdefault((name, entry.get("path") or name), []).append(entry) + return index + + def extra_alternatives(fe: dict, profiles: Mapping | None) -> list[dict]: """The versions the extra's own profile accepts at the extra's slot.""" if not profiles: diff --git a/tests/test_validated_choice.py b/tests/test_validated_choice.py index 3ad102bd3..ca7198b3c 100644 --- a/tests/test_validated_choice.py +++ b/tests/test_validated_choice.py @@ -296,5 +296,34 @@ class AWarningRejectsNothing(unittest.TestCase): short, _good, _db, index = self._setup(tmp, held_good=False) self.assertIsNone(check_file_validation(str(short), "OS12.rom", index, tmp)) + +class AlternativesAreLookedUpBySlot(unittest.TestCase): + """Each extra scanned its whole profile for the versions of its slot: + 33 million entries visited in one verify --all.""" + + def test_same_answer_one_pass_per_profile(self): + from unittest import mock + + import packextras + import validation + + files = [{"name": "DUKE3D.GRP", "size": n} for n in (1, 2)] + [ + {"name": "a.bin", "path": "dir/a.bin"}, {"name": "a.bin"}, "not an entry", + ] + profiles = {"raze": {"files": files}} + resolver = packextras.ExtraResolver({}, "bios", None, None, {}, None, profiles=profiles) + extras = [ + {"source_profile": "raze", "name": "DUKE3D.GRP"}, + {"source_profile": "raze", "name": "a.bin", "source_path": "dir/a.bin"}, + {"source_profile": "raze", "source_name": "a.bin", "name": "renamed"}, + {"source_profile": "other", "name": "a.bin"}, + ] + with mock.patch.object(validation, "slot_index", side_effect=validation.slot_index) as built: + for fe in extras: + self.assertEqual(resolver.alternatives(fe), + validation.extra_alternatives(fe, profiles)) + self.assertEqual(built.call_count, 2) + + if __name__ == "__main__": unittest.main()