From 32d415eda552c9acad6cd2c10a6b639abb6a420b Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Sun, 18 Oct 2026 16:03:43 +0200
Subject: [PATCH] fix: square the CRC chain once per length

---
 scripts/derivations.py       | 21 ++++++++++++---------
 scripts/provenance_report.py |  2 +-
 tests/test_derivations.py    | 18 ++++++++++++++++++
 3 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/scripts/derivations.py b/scripts/derivations.py
index 6d8b78cf..e3b5e30d 100644
--- a/scripts/derivations.py
+++ b/scripts/derivations.py
@@ -86,30 +86,33 @@ def crc32_combine(crc1: int, crc2: int, len2: int) -> int:
 def _shift_operator(len2: int) -> tuple[int, ...]:
     """The linear map zlib applies to crc(A) for len2 bytes appended.
 
-    Built once per length, column by column: chip sizes repeat, and
-    rebuilding the squares for every pair spent minutes in GF(2) products.
+    Built once per length, as zlib's crc32_combine walks it: the chain of
+    squared operators depends on len2 alone, so it is squared once and the
+    operators the set bits select are composed. Rebuilding the chain for
+    each of the 32 columns spent 57 s of the provenance step in GF(2)
+    products.
     """
-    return tuple(_shift(1 << bit, len2) for bit in range(32))
-
-
-def _shift(crc1: int, len2: int) -> int:
     odd = [0xEDB88320] + [1 << (n - 1) for n in range(1, 32)]
     even = _gf2_square(odd)
     odd = _gf2_square(even)
+    selected: list[list[int]] = []
     while True:
         even = _gf2_square(odd)
         if len2 & 1:
-            crc1 = _gf2_times(even, crc1)
+            selected.append(even)
         len2 >>= 1
         if not len2:
             break
         odd = _gf2_square(even)
         if len2 & 1:
-            crc1 = _gf2_times(odd, crc1)
+            selected.append(odd)
         len2 >>= 1
         if not len2:
             break
-    return crc1
+    columns = [1 << bit for bit in range(32)]
+    for matrix in selected:
+        columns = [_gf2_times(matrix, column) for column in columns]
+    return tuple(columns)
 
 
 def _read(held: dict, archives: dict[str, zipfile.ZipFile] | None = None) -> bytes | None:
diff --git a/scripts/provenance_report.py b/scripts/provenance_report.py
index 8da3c81f..16ea6120 100644
--- a/scripts/provenance_report.py
+++ b/scripts/provenance_report.py
@@ -178,7 +178,7 @@ def main() -> int:
         "--derive",
         action="store_true",
         help="Find missing entries a held dump contains (halves, word splits, "
-        "byte swaps, concatenations); reads the collection, about 90 s",
+        "byte swaps, concatenations); reads the collection, about 35 s",
     )
     parser.add_argument("--json", action="store_true", help="Full report as JSON")
     parser.add_argument(
diff --git a/tests/test_derivations.py b/tests/test_derivations.py
index e04c0ebb..38be8f5a 100644
--- a/tests/test_derivations.py
+++ b/tests/test_derivations.py
@@ -102,5 +102,23 @@ class ARomsetMemberIsASource(unittest.TestCase):
             )
 
 
+
+class CombineMatchesZlib(unittest.TestCase):
+    """The operator is composed once per length; it must stay zlib's."""
+
+    def test_against_zlib(self):
+        import os
+        import zlib
+
+        from derivations import crc32_combine
+
+        for size in (1, 2, 3, 7, 100, 4096, 65537, 1000003):
+            with self.subTest(size=size):
+                a, b = os.urandom(37), os.urandom(size)
+                self.assertEqual(
+                    crc32_combine(zlib.crc32(a), zlib.crc32(b), len(b)), zlib.crc32(a + b)
+                )
+
+
 if __name__ == "__main__":
     unittest.main()