From 727d1a22079c186c8cc816abfa45c03b4f208aa2 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sat, 17 Oct 2026 21:19:22 +0200 Subject: [PATCH] fix: derive catalog entries from romset members --- scripts/derivations.py | 152 ++++++++++++++++++++++++++++++-------- tests/test_derivations.py | 38 ++++++++++ 2 files changed, 158 insertions(+), 32 deletions(-) diff --git a/scripts/derivations.py b/scripts/derivations.py index 467c1f3d..6d8b78cf 100644 --- a/scripts/derivations.py +++ b/scripts/derivations.py @@ -10,8 +10,11 @@ one the source boards use; a match is by sha1, never by size or name. from __future__ import annotations +import array +import functools import hashlib import os +import zipfile from collections import defaultdict from collections.abc import Callable, Iterable @@ -26,7 +29,14 @@ def _swap16(data: bytes) -> bytes: def _words(data: bytes, start: int) -> bytes: - return b"".join(data[i:i + 2] for i in range(start * 2, len(data), 4)) + """Every other 16-bit word, from word `start`, bytes kept in order. + + Read as an array of 2-byte units and strided: joining slices one word + at a time made this pass take most of an hour over the romset chips. + """ + words = array.array("H") + words.frombytes(data[: len(data) - len(data) % 2]) + return words[start::2].tobytes() # (name, size of the output for an input of size n or None, transform) @@ -69,6 +79,20 @@ def crc32_combine(crc1: int, crc2: int, len2: int) -> int: """crc32 of A + B from crc32(A), crc32(B) and len(B), as zlib computes it.""" if len2 <= 0: return crc1 + return _gf2_times(_shift_operator(len2), crc1) ^ crc2 + + +@functools.lru_cache(maxsize=None) +def _shift_operator(len2: int) -> tuple[int, ...]: + """The linear map zlib applies to crc(A) for len2 bytes appended. + + Built once per length, column by column: chip sizes repeat, and + rebuilding the squares for every pair spent minutes in GF(2) products. + """ + return tuple(_shift(1 << bit, len2) for bit in range(32)) + + +def _shift(crc1: int, len2: int) -> int: odd = [0xEDB88320] + [1 << (n - 1) for n in range(1, 32)] even = _gf2_square(odd) odd = _gf2_square(even) @@ -85,22 +109,81 @@ def crc32_combine(crc1: int, crc2: int, len2: int) -> int: len2 >>= 1 if not len2: break - return crc1 ^ crc2 + return crc1 -def _read(path: str) -> bytes | None: +def _read(held: dict, archives: dict[str, zipfile.ZipFile] | None = None) -> bytes | None: + """The bytes of a held file, or of a member read out of its archive. + + `archives` keeps each archive open across its members: reopened for + every one, a MAME set of a hundred chips parsed its directory a + hundred times. + """ try: - return open(path, "rb").read() - except OSError: + if held.get("member"): + if archives is None: + with zipfile.ZipFile(held["archive"]) as zf: + return zf.read(held["member"]) + zf = archives.get(held["archive"]) + if zf is None: + zf = archives[held["archive"]] = zipfile.ZipFile(held["archive"]) + return zf.read(held["member"]) + with open(held["path"], "rb") as handle: + return handle.read() + except (OSError, KeyError, zipfile.BadZipFile): return None -def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]: - """Missing sha1 to the held file and transform that produce its bytes. +def _held_sources(db: dict) -> list[dict]: + """Every file the collection holds, the members of its archives included. - Only held files whose size can produce a wanted size are read, and a - concatenation is only tried between files of one directory: the chip - pairs a catalog splits come from one machine. + A romset holds chips a catalog lists on their own: cdi910.rom inside + cdi910.zip is the CD-i 910 BIOS byte-swapped, and the two halves of the + Salora Manager sit in manager.zip. Read from the top level alone, they + stayed MISSING. A member's group is its archive, as a file's is its + directory: the pairs a catalog splits come from one machine. + """ + sources: list[dict] = [] + for record in db.get("files", {}).values(): + path = record.get("path", "") + if not os.path.exists(path): + continue + if 0 < record.get("size", 0) <= MAX_SOURCE_SIZE: + sources.append({**record, "group": os.path.dirname(path)}) + if not path.lower().endswith(".zip"): + continue + try: + with zipfile.ZipFile(path) as zf: + infos = [i for i in zf.infolist() if not i.is_dir()] + except (OSError, zipfile.BadZipFile): + continue + for info in infos: + if 0 < info.file_size <= MAX_SOURCE_SIZE: + sources.append({ + "path": f"{path}/{info.filename}", "archive": path, + "member": info.filename, "size": info.file_size, + "crc32": f"{info.CRC:08x}", "group": path, + }) + return sources + + +def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]: + """Missing sha1 to the held file or member and the transform that + produce its bytes; archives stay open for the whole pass.""" + archives: dict[str, zipfile.ZipFile] = {} + try: + return _derivable(entries, db, archives) + finally: + for zf in archives.values(): + zf.close() + + +def _derivable( + entries: Iterable[dict], db: dict, archives: dict[str, zipfile.ZipFile] +) -> dict[str, dict]: + """Only held files whose size can produce a wanted size are read, and a + concatenation is only tried within one directory or one archive: the + chip pairs a catalog splits come from one machine. """ wanted: dict[tuple[int, str], str] = {} by_size_crc: dict[tuple[int, int], list[str]] = defaultdict(list) @@ -114,14 +197,13 @@ def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]: sizes = {size for size, _sha1 in wanted} found: dict[str, dict] = {} - files = [f for f in db.get("files", {}).values() - if 0 < f.get("size", 0) <= MAX_SOURCE_SIZE and os.path.exists(f["path"])] + files = _held_sources(db) for held in files: n = held["size"] useful = [t for t in TRANSFORMS if t[1](n) in sizes] if not useful: continue - data = _read(held["path"]) + data = _read(held, archives) if data is None: continue for name, out_size, transform in useful: @@ -133,25 +215,31 @@ def derivable(entries: Iterable[dict], db: dict) -> dict[str, dict]: by_dir: dict[str, list[dict]] = defaultdict(list) for held in files: if held.get("crc32"): - by_dir[os.path.dirname(held["path"])].append(held) + by_dir[held["group"]].append(held) for group in by_dir.values(): + # A pair is only worth a crc when its sizes add up to a wanted one: + # indexed by size, a group is walked once per wanted size instead + # of once per member, and a romset of a few hundred chips stays + # linear. + by_size: dict[int, list[dict]] = defaultdict(list) + for held in group: + by_size[held["size"]].append(held) for first in group: - for second in group: - if first is second: - continue - total = first["size"] + second["size"] - if total not in sizes: - continue - crc = crc32_combine(int(first["crc32"], 16), int(second["crc32"], 16), - second["size"]) - candidates = [s for s in by_size_crc.get((total, crc), []) if s not in found] - if not candidates: - continue - a, b = _read(first["path"]), _read(second["path"]) - if a is None or b is None: - continue - sha1 = hashlib.sha1(a + b).hexdigest() - if sha1 in candidates: - found[sha1] = {"from": [first["path"], second["path"]], - "transform": "concatenated"} + for total in sizes: + for second in by_size.get(total - first["size"], ()): + if first is second: + continue + crc = crc32_combine(int(first["crc32"], 16), + int(second["crc32"], 16), second["size"]) + candidates = [s for s in by_size_crc.get((total, crc), []) + if s not in found] + if not candidates: + continue + a, b = _read(first, archives), _read(second, archives) + if a is None or b is None: + continue + sha1 = hashlib.sha1(a + b).hexdigest() + if sha1 in candidates: + found[sha1] = {"from": [first["path"], second["path"]], + "transform": "concatenated"} return found diff --git a/tests/test_derivations.py b/tests/test_derivations.py index 2b1c8993..e04c0ebb 100644 --- a/tests/test_derivations.py +++ b/tests/test_derivations.py @@ -64,5 +64,43 @@ class HeldDumpsAnswerCatalogEntries(unittest.TestCase): self.assertEqual(derivations.derivable([_wanted(os.urandom(32))], db), {}) + +class ARomsetMemberIsASource(unittest.TestCase): + """cdi910.rom inside cdi910.zip is the CD-i 910 BIOS byte-swapped, and + manager.zip holds the two halves of the Salora Manager: read from the + top level alone, both stayed MISSING.""" + + def test_members_derive_and_concatenate(self): + import zipfile + + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + chip, low, high = os.urandom(64), os.urandom(32), os.urandom(32) + archive = root / "set.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("chip.rom", chip) + zf.writestr("01", low) + zf.writestr("23", high) + db = {"files": {"a" * 40: {"path": str(archive), "size": archive.stat().st_size}}} + swapped = derivations._swap16(chip) + found = derivations.derivable( + [_wanted(swapped), {**_wanted(low + high), "crc32": f"{zlib.crc32(low + high):08x}"}], + db, + ) + self.assertEqual(found[hashlib.sha1(swapped).hexdigest()]["transform"], "byte-swapped") + self.assertEqual(found[hashlib.sha1(low + high).hexdigest()]["transform"], "concatenated") + + def test_the_word_stride_and_crc_combine_are_exact(self): + data = os.urandom(4096) + slow = b"".join(data[i:i + 2] for i in range(0, len(data), 4)) + self.assertEqual(derivations._words(data, 0), slow) + for size in (1, 4096, 123457): + a, b = os.urandom(99), os.urandom(size) + self.assertEqual( + derivations.crc32_combine(zlib.crc32(a), zlib.crc32(b), size), + zlib.crc32(a + b), + ) + + if __name__ == "__main__": unittest.main()