fix: derive catalog entries from romset members

scripts/derivations.py

@@ -10,8 +10,11 @@ one the source boards use; a match is by sha1, never by size or name.
 
 from __future__ import annotations
 
+import array
+import functools
 import hashlib
 import os
+import zipfile
 from collections import defaultdict
 from collections.abc import Callable, Iterable
 
@@ -26,7 +29,14 @@ def _swap16(data: bytes) -> bytes:
 
 
 def _words(data: bytes, start: int) -> bytes:
-    return b"".join(data[i:i + 2] for i in range(start * 2, len(data), 4))
+    """Every other 16-bit word, from word `start`, bytes kept in order.
+
+    Read as an array of 2-byte units and strided: joining slices one word
+    at a time made this pass take most of an hour over the romset chips.
+    """
+    words = array.array("H")
+    words.frombytes(data[: len(data) - len(data) % 2])
+    return words[start::2].tobytes()
 
 
 # (name, size of the output for an input of size n or None, transform)
@@ -69,6 +79,20 @@ def crc32_combine(crc1: int, crc2: int, len2: int) -> int:
     """crc32 of A + B from crc32(A), crc32(B) and len(B), as zlib computes it."""
     if len2 <= 0:
         return crc1
+    return _gf2_times(_shift_operator(len2), crc1) ^ crc2
+
+
+@functools.lru_cache(maxsize=None)
+def _shift_operator(len2: int) -> tuple[int, ...]:
+    """The linear map zlib applies to crc(A) for len2 bytes appended.
+
+    Built once per length, column by column: chip sizes repeat, and
+    rebuilding the squares for every pair spent minutes in GF(2) products.
+    """
+    return tuple(_shift(1 << bit, len2) for bit in range(32))
+
+
+def _shift(crc1: int, len2: int) -> int:
     odd = [0xEDB88320] + [1 << (n - 1) for n in range(1, 32)]
     even = _gf2_square(odd)
     odd = _gf2_square(even)
@@ -85,22 +109,81 @@ def crc32_combine(crc1: int, crc2: int, len2: int) -> int:
         len2 >>= 1
         if not len2:
             break
-    return crc1 ^ crc2
+    return crc1
 
 
-def _read(path: str) -> bytes | None:
+def _read(held: dict, archives: dict[str, zipfile.ZipFile] | None = None) -> bytes | None:
+    """The bytes of a held file, or of a member read out of its archive.
+
+    `archives` keeps each archive open across its members: reopened for
+    every one, a MAME set of a hundred chips parsed its directory a
+    hundred times.
+    """
     try:
-        return open(path, "rb").read()
-    except OSError:
+        if held.get("member"):
+            if archives is None:
+                with zipfile.ZipFile(held["archive"]) as zf:
+                    return zf.read(held["member"])
+            zf = archives.get(held["archive"])
+            if zf is None:
+                zf = archives[held["archive"]] = zipfile.ZipFile(held["archive"])
+            return zf.read(held["member"])
+        with open(held["path"], "rb") as handle:
+            return handle.read()
+    except (OSError, KeyError, zipfile.BadZipFile):
         return None
 
 
-def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]:
-    """Missing sha1 to the held file and transform that produce its bytes.
+def _held_sources(db: dict) -> list[dict]:
+    """Every file the collection holds, the members of its archives included.
 
-    Only held files whose size can produce a wanted size are read, and a
-    concatenation is only tried between files of one directory: the chip
-    pairs a catalog splits come from one machine.
+    A romset holds chips a catalog lists on their own: cdi910.rom inside
+    cdi910.zip is the CD-i 910 BIOS byte-swapped, and the two halves of the
+    Salora Manager sit in manager.zip. Read from the top level alone, they
+    stayed MISSING. A member's group is its archive, as a file's is its
+    directory: the pairs a catalog splits come from one machine.
+    """
+    sources: list[dict] = []
+    for record in db.get("files", {}).values():
+        path = record.get("path", "")
+        if not os.path.exists(path):
+            continue
+        if 0 < record.get("size", 0) <= MAX_SOURCE_SIZE:
+            sources.append({**record, "group": os.path.dirname(path)})
+        if not path.lower().endswith(".zip"):
+            continue
+        try:
+            with zipfile.ZipFile(path) as zf:
+                infos = [i for i in zf.infolist() if not i.is_dir()]
+        except (OSError, zipfile.BadZipFile):
+            continue
+        for info in infos:
+            if 0 < info.file_size <= MAX_SOURCE_SIZE:
+                sources.append({
+                    "path": f"{path}/{info.filename}", "archive": path,
+                    "member": info.filename, "size": info.file_size,
+                    "crc32": f"{info.CRC:08x}", "group": path,
+                })
+    return sources
+
+
+def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]:
+    """Missing sha1 to the held file or member and the transform that
+    produce its bytes; archives stay open for the whole pass."""
+    archives: dict[str, zipfile.ZipFile] = {}
+    try:
+        return _derivable(entries, db, archives)
+    finally:
+        for zf in archives.values():
+            zf.close()
+
+
+def _derivable(
+    entries: Iterable[dict], db: dict, archives: dict[str, zipfile.ZipFile]
+) -> dict[str, dict]:
+    """Only held files whose size can produce a wanted size are read, and a
+    concatenation is only tried within one directory or one archive: the
+    chip pairs a catalog splits come from one machine.
     """
     wanted: dict[tuple[int, str], str] = {}
     by_size_crc: dict[tuple[int, int], list[str]] = defaultdict(list)
@@ -114,14 +197,13 @@ def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]:
     sizes = {size for size, _sha1 in wanted}
     found: dict[str, dict] = {}
 
-    files = [f for f in db.get("files", {}).values()
-             if 0 < f.get("size", 0) <= MAX_SOURCE_SIZE and os.path.exists(f["path"])]
+    files = _held_sources(db)
     for held in files:
         n = held["size"]
         useful = [t for t in TRANSFORMS if t[1](n) in sizes]
         if not useful:
             continue
-        data = _read(held["path"])
+        data = _read(held, archives)
         if data is None:
             continue
         for name, out_size, transform in useful:
@@ -133,25 +215,31 @@ def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]:
     by_dir: dict[str, list[dict]] = defaultdict(list)
     for held in files:
         if held.get("crc32"):
-            by_dir[os.path.dirname(held["path"])].append(held)
+            by_dir[held["group"]].append(held)
     for group in by_dir.values():
+        # A pair is only worth a crc when its sizes add up to a wanted one:
+        # indexed by size, a group is walked once per wanted size instead
+        # of once per member, and a romset of a few hundred chips stays
+        # linear.
+        by_size: dict[int, list[dict]] = defaultdict(list)
+        for held in group:
+            by_size[held["size"]].append(held)
         for first in group:
-            for second in group:
-                if first is second:
-                    continue
-                total = first["size"] + second["size"]
-                if total not in sizes:
-                    continue
-                crc = crc32_combine(int(first["crc32"], 16), int(second["crc32"], 16),
-                                    second["size"])
-                candidates = [s for s in by_size_crc.get((total, crc), []) if s not in found]
-                if not candidates:
-                    continue
-                a, b = _read(first["path"]), _read(second["path"])
-                if a is None or b is None:
-                    continue
-                sha1 = hashlib.sha1(a + b).hexdigest()
-                if sha1 in candidates:
-                    found[sha1] = {"from": [first["path"], second["path"]],
-                                   "transform": "concatenated"}
+            for total in sizes:
+                for second in by_size.get(total - first["size"], ()):
+                    if first is second:
+                        continue
+                    crc = crc32_combine(int(first["crc32"], 16),
+                                        int(second["crc32"], 16), second["size"])
+                    candidates = [s for s in by_size_crc.get((total, crc), [])
+                                  if s not in found]
+                    if not candidates:
+                        continue
+                    a, b = _read(first, archives), _read(second, archives)
+                    if a is None or b is None:
+                        continue
+                    sha1 = hashlib.sha1(a + b).hexdigest()
+                    if sha1 in candidates:
+                        found[sha1] = {"from": [first["path"], second["path"]],
+                                       "transform": "concatenated"}
     return found

tests/test_derivations.py

@@ -64,5 +64,43 @@ class HeldDumpsAnswerCatalogEntries(unittest.TestCase):
         self.assertEqual(derivations.derivable([_wanted(os.urandom(32))], db), {})
 
 
+
+class ARomsetMemberIsASource(unittest.TestCase):
+    """cdi910.rom inside cdi910.zip is the CD-i 910 BIOS byte-swapped, and
+    manager.zip holds the two halves of the Salora Manager: read from the
+    top level alone, both stayed MISSING."""
+
+    def test_members_derive_and_concatenate(self):
+        import zipfile
+
+        root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
+        self.addCleanup(shutil.rmtree, root, True)
+        chip, low, high = os.urandom(64), os.urandom(32), os.urandom(32)
+        archive = root / "set.zip"
+        with zipfile.ZipFile(archive, "w") as zf:
+            zf.writestr("chip.rom", chip)
+            zf.writestr("01", low)
+            zf.writestr("23", high)
+        db = {"files": {"a" * 40: {"path": str(archive), "size": archive.stat().st_size}}}
+        swapped = derivations._swap16(chip)
+        found = derivations.derivable(
+            [_wanted(swapped), {**_wanted(low + high), "crc32": f"{zlib.crc32(low + high):08x}"}],
+            db,
+        )
+        self.assertEqual(found[hashlib.sha1(swapped).hexdigest()]["transform"], "byte-swapped")
+        self.assertEqual(found[hashlib.sha1(low + high).hexdigest()]["transform"], "concatenated")
+
+    def test_the_word_stride_and_crc_combine_are_exact(self):
+        data = os.urandom(4096)
+        slow = b"".join(data[i:i + 2] for i in range(0, len(data), 4))
+        self.assertEqual(derivations._words(data, 0), slow)
+        for size in (1, 4096, 123457):
+            a, b = os.urandom(99), os.urandom(size)
+            self.assertEqual(
+                derivations.crc32_combine(zlib.crc32(a), zlib.crc32(b), size),
+                zlib.crc32(a + b),
+            )
+
+
 if __name__ == "__main__":
     unittest.main()