fix: square the CRC chain once per length

scripts/derivations.py

@@ -86,30 +86,33 @@ def crc32_combine(crc1: int, crc2: int, len2: int) -> int:
 def _shift_operator(len2: int) -> tuple[int, ...]:
     """The linear map zlib applies to crc(A) for len2 bytes appended.
 
-    Built once per length, column by column: chip sizes repeat, and
-    rebuilding the squares for every pair spent minutes in GF(2) products.
+    Built once per length, as zlib's crc32_combine walks it: the chain of
+    squared operators depends on len2 alone, so it is squared once and the
+    operators the set bits select are composed. Rebuilding the chain for
+    each of the 32 columns spent 57 s of the provenance step in GF(2)
+    products.
     """
-    return tuple(_shift(1 << bit, len2) for bit in range(32))
-
-
-def _shift(crc1: int, len2: int) -> int:
     odd = [0xEDB88320] + [1 << (n - 1) for n in range(1, 32)]
     even = _gf2_square(odd)
     odd = _gf2_square(even)
+    selected: list[list[int]] = []
     while True:
         even = _gf2_square(odd)
         if len2 & 1:
-            crc1 = _gf2_times(even, crc1)
+            selected.append(even)
         len2 >>= 1
         if not len2:
             break
         odd = _gf2_square(even)
         if len2 & 1:
-            crc1 = _gf2_times(odd, crc1)
+            selected.append(odd)
         len2 >>= 1
         if not len2:
             break
-    return crc1
+    columns = [1 << bit for bit in range(32)]
+    for matrix in selected:
+        columns = [_gf2_times(matrix, column) for column in columns]
+    return tuple(columns)
 
 
 def _read(held: dict, archives: dict[str, zipfile.ZipFile] | None = None) -> bytes | None:

scripts/provenance_report.py

@@ -178,7 +178,7 @@ def main() -> int:
         "--derive",
         action="store_true",
         help="Find missing entries a held dump contains (halves, word splits, "
-        "byte swaps, concatenations); reads the collection, about 90 s",
+        "byte swaps, concatenations); reads the collection, about 35 s",
     )
     parser.add_argument("--json", action="store_true", help="Full report as JSON")
     parser.add_argument(

tests/test_derivations.py

@@ -102,5 +102,23 @@ class ARomsetMemberIsASource(unittest.TestCase):
             )
 
 
+
+class CombineMatchesZlib(unittest.TestCase):
+    """The operator is composed once per length; it must stay zlib's."""
+
+    def test_against_zlib(self):
+        import os
+        import zlib
+
+        from derivations import crc32_combine
+
+        for size in (1, 2, 3, 7, 100, 4096, 65537, 1000003):
+            with self.subTest(size=size):
+                a, b = os.urandom(37), os.urandom(size)
+                self.assertEqual(
+                    crc32_combine(zlib.crc32(a), zlib.crc32(b), len(b)), zlib.crc32(a + b)
+                )
+
+
 if __name__ == "__main__":
     unittest.main()