chore: merge round 2 fixes (perf)

tests/test_archive_choice.py

@@ -409,5 +409,50 @@ class ALooseEntryNamingItsArchive(unittest.TestCase):
         self.assertEqual(report["total_files"], 1)
 
 
+class ArchiveMembersIndexOnce(unittest.TestCase):
+    """with_archive_members asks once per entry of a profile: scanning the
+    whole list for each walked 38 million entries in one verify --all."""
+
+    class CountingList(list):
+        walks = 0
+
+        def __iter__(self):
+            type(self).walks += 1
+            return super().__iter__()
+
+    def entries(self) -> list[dict]:
+        files = self.CountingList()
+        self.CountingList.walks = 0
+        for n in range(300):
+            files.append({"name": f"m{n}.rom", "archive": f"set{n % 30}.zip",
+                          "crc32": f"{n:08X}", "required": n % 3 != 0})
+        files.append({"name": "loose.bin", "crc32": "deadbeef"})
+        files.append({"name": "nocrc.rom", "archive": "set0.zip"})
+        return files
+
+    @staticmethod
+    def scanned(archive, files) -> list[dict]:
+        return [{"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
+                for f in list.__iter__(files) if f.get("archive") == archive and f.get("crc32")]
+
+    def test_a_profile_list_is_walked_once_and_answers_as_a_scan(self):
+        from common import archive_members
+
+        files = self.entries()
+        for archive in [f"set{n}.zip" for n in range(30)] + ["absent.zip", "", None]:
+            with self.subTest(archive=archive):
+                self.assertEqual(archive_members(archive, files), self.scanned(archive, files))
+        self.assertEqual(self.CountingList.walks, 1)
+
+    def test_a_list_that_grew_is_indexed_again(self):
+        from common import archive_members
+
+        files = self.entries()
+        archive_members("set1.zip", files)
+        files.append({"name": "late.rom", "archive": "set1.zip", "crc32": "ABCDEF01"})
+        self.assertEqual(archive_members("set1.zip", files), self.scanned("set1.zip", files))
+        self.assertIn({"crc32": "abcdef01", "required": True}, archive_members("set1.zip", files))
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_crypto_verify.py

@@ -19,6 +19,7 @@ import shutil
 import sys
 import tempfile
 import unittest
+import unittest.mock
 from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
@@ -329,5 +330,44 @@ class KeysFile(unittest.TestCase):
         self.assertIsNone(crypto_verify.find_keys_file(self.tmp))
 
 
+class OneFileIsVerifiedOncePerContent(unittest.TestCase):
+    """verify --all ran the 3DS checks of the same five files once per
+    platform: 35 runs of pure-Python curve and RSA work for 5 answers."""
+
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.bios = Path(self._tmp.name)
+        target = self.bios / "Nintendo" / "3DS"
+        target.mkdir(parents=True)
+        (target / "aes_keys.txt").write_text(":AES\n")
+        self.file = target / "otp.bin"
+        self.file.write_bytes(b"\x00" * 0x100)
+        self.calls: list[bytes] = []
+
+        def verifier(path, keys):
+            data = Path(path).read_bytes()
+            self.calls.append(data)
+            return data[0] == 1, "first byte is not 1"
+
+        patcher = unittest.mock.patch.dict(crypto_verify._CRYPTO_VERIFIERS, {"otp.bin": verifier})
+        patcher.start()
+        self.addCleanup(patcher.stop)
+        self.addCleanup(crypto_verify._crypto_cache.clear)
+        self.addCleanup(self._tmp.cleanup)
+
+    def check(self) -> str | None:
+        return crypto_verify.check_crypto_validation(str(self.file), "otp.bin", str(self.bios))
+
+    def test_the_same_file_is_verified_once(self):
+        self.assertEqual([self.check() for _ in range(7)], ["first byte is not 1"] * 7)
+        self.assertEqual(len(self.calls), 1)
+
+    def test_a_rewritten_file_is_verified_again(self):
+        self.assertEqual(self.check(), "first byte is not 1")
+        self.file.write_bytes(b"\x01" + b"\x00" * 0x1FF)
+        self.assertIsNone(self.check())
+        self.assertEqual(len(self.calls), 2)
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_dedup.py

@@ -309,5 +309,37 @@ class TheNoDedupListIsPinned(unittest.TestCase):
         self.assertEqual(expected - set(NODEDUP_DIRS), set())
 
 
+class OnlyFilesSharingASizeAreHashed(_Bios):
+    """Equal files have equal sizes. Hashing all 9 442 files with five
+    digests took 33 s for 278 groups that a sha1 of the 4 543 files sharing
+    a size finds the same."""
+
+    def test_a_file_alone_at_its_size_is_not_read(self):
+        from unittest import mock
+
+        self.write("A/one.bin")
+        self.write("B/one.bin")
+        self.write("C/other.bin", b"a payload of its own length")
+        self.write("D/same_size.bin", b"IDENTICAL PAYLOAX")
+        calls: list[tuple[str, frozenset | None]] = []
+        real = dedup.compute_hashes
+
+        def spy(path, algorithms=None, cache=True):
+            calls.append((os.path.relpath(path, self.bios), algorithms))
+            return real(path, algorithms, cache)
+
+        with mock.patch.object(dedup, "compute_hashes", spy):
+            groups = dedup.scan_duplicates(str(self.bios))
+        self.assertEqual(
+            sorted(path for path, _algos in calls),
+            sorted(os.path.join(*p.split("/")) for p in ("A/one.bin", "B/one.bin", "D/same_size.bin")),
+        )
+        self.assertTrue(all(algos == frozenset({"sha1"}) for _path, algos in calls))
+        multi = [paths for paths in groups.values() if len(paths) > 1]
+        self.assertEqual(len(multi), 1)
+        self.assertEqual(sorted(os.path.relpath(p, self.bios) for p in multi[0]),
+                         sorted(os.path.join(*p.split("/")) for p in ("A/one.bin", "B/one.bin")))
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_derivations.py

@@ -13,6 +13,7 @@ import shutil
 import sys
 import tempfile
 import unittest
+import unittest.mock
 import zlib
 from pathlib import Path
 
@@ -103,6 +104,48 @@ class ARomsetMemberIsASource(unittest.TestCase):
 
 
 
+class OneContentIsTransformedOnce(unittest.TestCase):
+    """A MAME chip sits in many archives: 18 924 reads for 7 407 contents, and
+    each read went through every transform again."""
+
+    def test_a_repeated_chip_is_transformed_once_and_keeps_its_first_path(self):
+        import zipfile
+
+        root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
+        self.addCleanup(shutil.rmtree, root, True)
+        chip = os.urandom(64)
+        files = {}
+        for name in ("a.zip", "b.zip", "c.zip"):
+            archive = root / name
+            with zipfile.ZipFile(archive, "w") as zf:
+                zf.writestr("chip.rom", chip)
+            files[name] = {"path": str(archive), "size": archive.stat().st_size}
+        calls = []
+
+        def counted(transform):
+            def run(data):
+                calls.append(data)
+                return transform(data)
+            return run
+
+        transforms = tuple((name, size, counted(fn)) for name, size, fn in derivations.TRANSFORMS)
+        swapped = derivations._swap16(chip)
+        with unittest.mock.patch.object(derivations, "TRANSFORMS", transforms):
+            found = derivations.derivable([_wanted(swapped)], {"files": files})
+        self.assertEqual(found[hashlib.sha1(swapped).hexdigest()]["from"],
+                         [f"{root / 'a.zip'}/chip.rom"])
+        useful = [t for t in transforms if t[1](len(chip)) == len(chip)]
+        self.assertEqual(len(calls), len(useful))
+
+    def test_the_shared_power_operators_compose_to_zlib_for_every_bit(self):
+        a = os.urandom(17)
+        for size in [1 << bit for bit in range(25)] + [(1 << 25) - 1, 3624, 5]:
+            with self.subTest(size=size):
+                b = os.urandom(size)
+                self.assertEqual(derivations.crc32_combine(zlib.crc32(a), zlib.crc32(b), size),
+                                 zlib.crc32(b, zlib.crc32(a)))
+
+
 class CombineMatchesZlib(unittest.TestCase):
     """The operator is composed once per length, and it must stay zlib's."""