From cdab4e780879cc6f2d40492b2e6dfae83f085464 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sun, 18 Oct 2026 00:00:04 +0200 Subject: [PATCH] refactor: keep romset build digests between runs --- scripts/romset_recipes.py | 77 ++++++++++++++++++++++++++++++++---- tests/test_romset_recipes.py | 33 ++++++++++++++++ 2 files changed, 103 insertions(+), 7 deletions(-) diff --git a/scripts/romset_recipes.py b/scripts/romset_recipes.py index dd7dec33..c5887649 100644 --- a/scripts/romset_recipes.py +++ b/scripts/romset_recipes.py @@ -121,8 +121,15 @@ class AtomPool: hundreds of megabytes and a whole-pool preload buys nothing. """ - def __init__(self, bios_dir: Path): + def __init__(self, bios_dir: Path, digest_cache: Path | None = None): self._index: dict[str, tuple[Path, str]] = {} + # (md5, size) of each build, kept across runs: TorrentZip bytes are a + # function of the members alone, so a recipe hashed once is known + # until the writer changes. Rebuilding the 1 221 archives took 70 s + # of every pipeline run, 63 s of it in zlib. + self._digest_cache = digest_cache + self._digests: dict[str, list] = _read_digests(digest_cache) + self._digests_changed = False # A recipe is built once: identification and reconstruction ask for the # same archives, and every build re-reads and re-deflates its ROMs. self._built: dict[tuple[tuple[str, str], ...], bytes | None] = {} @@ -149,6 +156,31 @@ class AtomPool: with zipfile.ZipFile(archive) as handle: return handle.read(member) + def digest(self, recipe: Recipe) -> tuple[str, int] | None: + """(md5, size) of the recipe's TorrentZip build, None if incomplete.""" + if not all(crc in self for _name, crc in recipe): + return None + key = _recipe_key(recipe) + known = self._digests.get(key) + if known is None: + blob = self.build(recipe) + if blob is None: + return None + known = self._digests[key] = [hashlib.md5(blob).hexdigest(), len(blob)] + self._digests_changed = True + return known[0], known[1] + + def save_digests(self) -> None: + """Write the digest cache back, when this run learned anything.""" + if self._digest_cache is None or not self._digests_changed: + return + from artifacts import write_text_atomic # noqa: PLC0415 + + self._digest_cache.parent.mkdir(parents=True, exist_ok=True) + write_text_atomic( + str(self._digest_cache), json.dumps(self._digests, sort_keys=True) + "\n" + ) + def build(self, recipe: Recipe) -> bytes | None: """Return the TorrentZip bytes for *recipe*, or None if incomplete.""" key = tuple(recipe) @@ -162,6 +194,33 @@ class AtomPool: return self._built[key] +DIGEST_CACHE = Path(__file__).resolve().parent.parent / ".cache" / "romset_digests.json" +# The writer's own source is part of every key: a change to how TorrentZip +# bytes are laid out invalidates every digest it produced. +_WRITER = hashlib.sha256( + (Path(__file__).resolve().parent / "torrentzip.py").read_bytes() +).hexdigest()[:16] + + +def _recipe_key(recipe: Recipe) -> str: + members = json.dumps([[name, crc.lower()] for name, crc in recipe]) + return hashlib.sha256(f"{_WRITER}:{members}".encode()).hexdigest() + + +def _read_digests(path: Path | None) -> dict[str, list]: + if path is None: + return {} + try: + with open(path, encoding="utf-8") as handle: + data = json.load(handle) + except FileNotFoundError: + return {} + except (OSError, ValueError) as exc: + print(f" digest cache {path} unreadable, rebuilding: {exc}", file=sys.stderr) + return {} + return data if isinstance(data, dict) else {} + + def identify_archive( path: Path, recipes: dict[str, Recipe], pool: AtomPool ) -> str | None: @@ -175,11 +234,11 @@ def identify_archive( if held is None: return None original = path.read_bytes() + md5, size = hashlib.md5(original).hexdigest(), len(original) for label, recipe in sorted(recipes.items()): if sorted((name, crc.lower()) for name, crc in recipe) != held: continue - blob = pool.build(recipe) - if blob is not None and blob == original: + if pool.digest(recipe) == (md5, size): return label return None @@ -203,9 +262,11 @@ def reconstruct( """Return the (label, bytes) whose TorrentZip build matches *target_md5*.""" target = target_md5.lower() for label, recipe in sorted(recipes.items()): - blob = pool.build(recipe) - if blob is not None and hashlib.md5(blob).hexdigest() == target: - return label, blob + known = pool.digest(recipe) + if known is not None and known[0] == target: + blob = pool.build(recipe) + if blob is not None and hashlib.md5(blob).hexdigest() == target: + return label, blob return None @@ -405,7 +466,7 @@ def main() -> int: profile_recipes = collect_recipes(profiles) dat_recipes = load_dat_recipes(args.dat_recipes) recipes = merge_recipes(profile_recipes, dat_recipes) - pool = AtomPool(Path(args.bios_dir)) + pool = AtomPool(Path(args.bios_dir), DIGEST_CACHE) result: dict = { "recipes": sum(len(v) for v in recipes.values()), @@ -426,6 +487,8 @@ def main() -> int: Path(args.bios_dir), db, ) + pool.save_digests() + if args.json: print(json.dumps(result, indent=2, sort_keys=True)) return 0 diff --git a/tests/test_romset_recipes.py b/tests/test_romset_recipes.py index 6ec5d4ec..ce03282c 100644 --- a/tests/test_romset_recipes.py +++ b/tests/test_romset_recipes.py @@ -582,11 +582,44 @@ class IdentificationReadsTheDirectoryFirst(unittest.TestCase): self.built.append(recipe) return build_torrentzip([(name, rom) for name, _crc in recipe]) + def digest(self, recipe): + import hashlib # noqa: PLC0415 + + blob = self.build(recipe) + return hashlib.md5(blob).hexdigest(), len(blob) + pool = Pool() label = romset_recipes.identify_archive( archive, {"other": [("b.rom", crc)], "match": [("a.rom", crc)]}, pool) self.assertEqual(label, "match") self.assertEqual(pool.built, [[("a.rom", crc)]]) + +class BuildDigestsOutliveTheRun(unittest.TestCase): + """Step 1c rebuilt every recipe at DEFLATE 9 on each pipeline run (70 s) + for bytes that depend only on the members.""" + + def test_a_second_pool_reads_the_digest_without_building(self): + import hashlib + + from unittest import mock + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp: + bios = Path(tmp) / "bios" + bios.mkdir() + with zipfile.ZipFile(bios / "source.zip", "w") as archive: + archive.writestr("a.rom", ROM_A) + cache = Path(tmp) / "digests.json" + recipe = [("a.rom", _crc(ROM_A))] + first = AtomPool(bios, cache) + digest = first.digest(recipe) + blob = first.build(recipe) + self.assertEqual(digest, (hashlib.md5(blob).hexdigest(), len(blob))) + first.save_digests() + second = AtomPool(bios, cache) + with mock.patch.object(second, "build", side_effect=AssertionError("rebuilt")): + self.assertEqual(second.digest(recipe), digest) + self.assertIsNone(second.digest([("x.rom", "deadbeef")])) + if __name__ == "__main__": unittest.main()