refactor: keep romset build digests between runs

scripts/romset_recipes.py

@@ -121,8 +121,15 @@ class AtomPool:
     hundreds of megabytes and a whole-pool preload buys nothing.
     """
 
-    def __init__(self, bios_dir: Path):
+    def __init__(self, bios_dir: Path, digest_cache: Path | None = None):
         self._index: dict[str, tuple[Path, str]] = {}
+        # (md5, size) of each build, kept across runs: TorrentZip bytes are a
+        # function of the members alone, so a recipe hashed once is known
+        # until the writer changes. Rebuilding the 1 221 archives took 70 s
+        # of every pipeline run, 63 s of it in zlib.
+        self._digest_cache = digest_cache
+        self._digests: dict[str, list] = _read_digests(digest_cache)
+        self._digests_changed = False
         # A recipe is built once: identification and reconstruction ask for the
         # same archives, and every build re-reads and re-deflates its ROMs.
         self._built: dict[tuple[tuple[str, str], ...], bytes | None] = {}
@@ -149,6 +156,31 @@ class AtomPool:
         with zipfile.ZipFile(archive) as handle:
             return handle.read(member)
 
+    def digest(self, recipe: Recipe) -> tuple[str, int] | None:
+        """(md5, size) of the recipe's TorrentZip build, None if incomplete."""
+        if not all(crc in self for _name, crc in recipe):
+            return None
+        key = _recipe_key(recipe)
+        known = self._digests.get(key)
+        if known is None:
+            blob = self.build(recipe)
+            if blob is None:
+                return None
+            known = self._digests[key] = [hashlib.md5(blob).hexdigest(), len(blob)]
+            self._digests_changed = True
+        return known[0], known[1]
+
+    def save_digests(self) -> None:
+        """Write the digest cache back, when this run learned anything."""
+        if self._digest_cache is None or not self._digests_changed:
+            return
+        from artifacts import write_text_atomic  # noqa: PLC0415
+
+        self._digest_cache.parent.mkdir(parents=True, exist_ok=True)
+        write_text_atomic(
+            str(self._digest_cache), json.dumps(self._digests, sort_keys=True) + "\n"
+        )
+
     def build(self, recipe: Recipe) -> bytes | None:
         """Return the TorrentZip bytes for *recipe*, or None if incomplete."""
         key = tuple(recipe)
@@ -162,6 +194,33 @@ class AtomPool:
         return self._built[key]
 
 
+DIGEST_CACHE = Path(__file__).resolve().parent.parent / ".cache" / "romset_digests.json"
+# The writer's own source is part of every key: a change to how TorrentZip
+# bytes are laid out invalidates every digest it produced.
+_WRITER = hashlib.sha256(
+    (Path(__file__).resolve().parent / "torrentzip.py").read_bytes()
+).hexdigest()[:16]
+
+
+def _recipe_key(recipe: Recipe) -> str:
+    members = json.dumps([[name, crc.lower()] for name, crc in recipe])
+    return hashlib.sha256(f"{_WRITER}:{members}".encode()).hexdigest()
+
+
+def _read_digests(path: Path | None) -> dict[str, list]:
+    if path is None:
+        return {}
+    try:
+        with open(path, encoding="utf-8") as handle:
+            data = json.load(handle)
+    except FileNotFoundError:
+        return {}
+    except (OSError, ValueError) as exc:
+        print(f"  digest cache {path} unreadable, rebuilding: {exc}", file=sys.stderr)
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
 def identify_archive(
     path: Path, recipes: dict[str, Recipe], pool: AtomPool
 ) -> str | None:
@@ -175,11 +234,11 @@ def identify_archive(
     if held is None:
         return None
     original = path.read_bytes()
+    md5, size = hashlib.md5(original).hexdigest(), len(original)
     for label, recipe in sorted(recipes.items()):
         if sorted((name, crc.lower()) for name, crc in recipe) != held:
             continue
-        blob = pool.build(recipe)
+        if pool.digest(recipe) == (md5, size):
-        if blob is not None and blob == original:
             return label
     return None
 
@@ -203,9 +262,11 @@ def reconstruct(
     """Return the (label, bytes) whose TorrentZip build matches *target_md5*."""
     target = target_md5.lower()
     for label, recipe in sorted(recipes.items()):
-        blob = pool.build(recipe)
+        known = pool.digest(recipe)
-        if blob is not None and hashlib.md5(blob).hexdigest() == target:
+        if known is not None and known[0] == target:
-            return label, blob
+            blob = pool.build(recipe)
+            if blob is not None and hashlib.md5(blob).hexdigest() == target:
+                return label, blob
     return None
 
 
@@ -405,7 +466,7 @@ def main() -> int:
     profile_recipes = collect_recipes(profiles)
     dat_recipes = load_dat_recipes(args.dat_recipes)
     recipes = merge_recipes(profile_recipes, dat_recipes)
-    pool = AtomPool(Path(args.bios_dir))
+    pool = AtomPool(Path(args.bios_dir), DIGEST_CACHE)
 
     result: dict = {
         "recipes": sum(len(v) for v in recipes.values()),
@@ -426,6 +487,8 @@ def main() -> int:
                 Path(args.bios_dir), db,
             )
 
+    pool.save_digests()
+
     if args.json:
         print(json.dumps(result, indent=2, sort_keys=True))
         return 0

tests/test_romset_recipes.py

@@ -582,11 +582,44 @@ class IdentificationReadsTheDirectoryFirst(unittest.TestCase):
                     self.built.append(recipe)
                     return build_torrentzip([(name, rom) for name, _crc in recipe])
 
+                def digest(self, recipe):
+                    import hashlib  # noqa: PLC0415
+
+                    blob = self.build(recipe)
+                    return hashlib.md5(blob).hexdigest(), len(blob)
+
             pool = Pool()
             label = romset_recipes.identify_archive(
                 archive, {"other": [("b.rom", crc)], "match": [("a.rom", crc)]}, pool)
         self.assertEqual(label, "match")
         self.assertEqual(pool.built, [[("a.rom", crc)]])
 
+
+class BuildDigestsOutliveTheRun(unittest.TestCase):
+    """Step 1c rebuilt every recipe at DEFLATE 9 on each pipeline run (70 s)
+    for bytes that depend only on the members."""
+
+    def test_a_second_pool_reads_the_digest_without_building(self):
+        import hashlib
+
+        from unittest import mock
+
+        with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp:
+            bios = Path(tmp) / "bios"
+            bios.mkdir()
+            with zipfile.ZipFile(bios / "source.zip", "w") as archive:
+                archive.writestr("a.rom", ROM_A)
+            cache = Path(tmp) / "digests.json"
+            recipe = [("a.rom", _crc(ROM_A))]
+            first = AtomPool(bios, cache)
+            digest = first.digest(recipe)
+            blob = first.build(recipe)
+            self.assertEqual(digest, (hashlib.md5(blob).hexdigest(), len(blob)))
+            first.save_digests()
+            second = AtomPool(bios, cache)
+            with mock.patch.object(second, "build", side_effect=AssertionError("rebuilt")):
+                self.assertEqual(second.digest(recipe), digest)
+            self.assertIsNone(second.digest([("x.rom", "deadbeef")]))
+
 if __name__ == "__main__":
     unittest.main()