diff --git a/scripts/derivations.py b/scripts/derivations.py index 4a71b91e..5be36596 100644 --- a/scripts/derivations.py +++ b/scripts/derivations.py @@ -82,36 +82,36 @@ def crc32_combine(crc1: int, crc2: int, len2: int) -> int: return _gf2_times(_shift_operator(len2), crc1) ^ crc2 +@functools.lru_cache(maxsize=None) +def _byte_shift(power: int) -> tuple[int, ...]: + """The operator for 2**power zero bytes appended, as zlib squares it: + three squarings of the one-bit operator give one byte, and each power + is the square of the one before.""" + if power: + return tuple(_gf2_square(list(_byte_shift(power - 1)))) + operator = [0xEDB88320] + [1 << (n - 1) for n in range(1, 32)] + for _ in range(3): + operator = _gf2_square(operator) + return tuple(operator) + + @functools.lru_cache(maxsize=None) def _shift_operator(len2: int) -> tuple[int, ...]: """The linear map zlib applies to crc(A) for len2 bytes appended. - Built once per length, as zlib's crc32_combine walks it: the chain of - squared operators depends on len2 alone, so it is squared once and the - operators the set bits select are composed. Rebuilding the chain for - each of the 32 columns spent 57 s of the provenance step in GF(2) - products. + The product of the power-of-two operators the set bits of len2 select. + Those powers are shared by every length, so each is squared once for the + whole pass: squaring the chain again for each of the 3 600 lengths a + romset pairs spent most of the crc work in GF(2) products. """ - odd = [0xEDB88320] + [1 << (n - 1) for n in range(1, 32)] - even = _gf2_square(odd) - odd = _gf2_square(even) - selected: list[list[int]] = [] - while True: - even = _gf2_square(odd) - if len2 & 1: - selected.append(even) - len2 >>= 1 - if not len2: - break - odd = _gf2_square(even) - if len2 & 1: - selected.append(odd) - len2 >>= 1 - if not len2: - break columns = [1 << bit for bit in range(32)] - for matrix in selected: - columns = [_gf2_times(matrix, column) for column in columns] + power = 0 + while len2: + if len2 & 1: + matrix = _byte_shift(power) + columns = [_gf2_times(matrix, column) for column in columns] + len2 >>= 1 + power += 1 return tuple(columns) @@ -201,6 +201,10 @@ def _derivable( found: dict[str, dict] = {} files = _held_sources(db) + # The same chip sits in many archives. Its transforms are a function of + # its bytes alone, and the first path read keeps the match, so a content + # already transformed has nothing left to find. + transformed: set[str] = set() for held in files: n = held["size"] useful = [t for t in TRANSFORMS if t[1](n) in sizes] @@ -209,31 +213,42 @@ def _derivable( data = _read(held, archives) if data is None: continue + content = hashlib.sha1(data).hexdigest() + if content in transformed: + continue + transformed.add(content) for name, out_size, transform in useful: out = transform(data) sha1 = wanted.get((out_size(n), hashlib.sha1(out).hexdigest())) if sha1 and sha1 not in found: found[sha1] = {"from": [held["path"]], "transform": name} + # Romsets repeat the same chip pairs: a combined crc is computed once. + combined: dict[tuple[str, str, int], int] = {} by_dir: dict[str, list[dict]] = defaultdict(list) for held in files: if held.get("crc32"): by_dir[held["group"]].append(held) for group in by_dir.values(): # A pair is only worth a crc when its sizes add up to a wanted one: - # indexed by size, a group is walked once per wanted size instead - # of once per member, and a romset of a few hundred chips stays - # linear. + # indexed by size, a member is paired once per size the group + # holds, and a romset of a few hundred chips stays linear. by_size: dict[int, list[dict]] = defaultdict(list) for held in group: by_size[held["size"]].append(held) for first in group: - for total in sizes: - for second in by_size.get(total - first["size"], ()): + for second_size, seconds in by_size.items(): + total = first["size"] + second_size + if total not in sizes: + continue + for second in seconds: if first is second: continue - crc = crc32_combine(int(first["crc32"], 16), - int(second["crc32"], 16), second["size"]) + key = (first["crc32"], second["crc32"], second["size"]) + crc = combined.get(key) + if crc is None: + crc = combined[key] = crc32_combine( + int(first["crc32"], 16), int(second["crc32"], 16), second["size"]) candidates = [s for s in by_size_crc.get((total, crc), []) if s not in found] if not candidates: diff --git a/tests/test_derivations.py b/tests/test_derivations.py index a346224e..580bf32e 100644 --- a/tests/test_derivations.py +++ b/tests/test_derivations.py @@ -13,6 +13,7 @@ import shutil import sys import tempfile import unittest +import unittest.mock import zlib from pathlib import Path @@ -103,6 +104,48 @@ class ARomsetMemberIsASource(unittest.TestCase): +class OneContentIsTransformedOnce(unittest.TestCase): + """A MAME chip sits in many archives: 18 924 reads for 7 407 contents, and + each read went through every transform again.""" + + def test_a_repeated_chip_is_transformed_once_and_keeps_its_first_path(self): + import zipfile + + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + chip = os.urandom(64) + files = {} + for name in ("a.zip", "b.zip", "c.zip"): + archive = root / name + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("chip.rom", chip) + files[name] = {"path": str(archive), "size": archive.stat().st_size} + calls = [] + + def counted(transform): + def run(data): + calls.append(data) + return transform(data) + return run + + transforms = tuple((name, size, counted(fn)) for name, size, fn in derivations.TRANSFORMS) + swapped = derivations._swap16(chip) + with unittest.mock.patch.object(derivations, "TRANSFORMS", transforms): + found = derivations.derivable([_wanted(swapped)], {"files": files}) + self.assertEqual(found[hashlib.sha1(swapped).hexdigest()]["from"], + [f"{root / 'a.zip'}/chip.rom"]) + useful = [t for t in transforms if t[1](len(chip)) == len(chip)] + self.assertEqual(len(calls), len(useful)) + + def test_the_shared_power_operators_compose_to_zlib_for_every_bit(self): + a = os.urandom(17) + for size in [1 << bit for bit in range(25)] + [(1 << 25) - 1, 3624, 5]: + with self.subTest(size=size): + b = os.urandom(size) + self.assertEqual(derivations.crc32_combine(zlib.crc32(a), zlib.crc32(b), size), + zlib.crc32(b, zlib.crc32(a))) + + class CombineMatchesZlib(unittest.TestCase): """The operator is composed once per length, and it must stay zlib's."""