diff --git a/scripts/romset_recipes.py b/scripts/romset_recipes.py index f44a8840..d1b58d64 100644 --- a/scripts/romset_recipes.py +++ b/scripts/romset_recipes.py @@ -165,15 +165,38 @@ class AtomPool: def identify_archive( path: Path, recipes: dict[str, Recipe], pool: AtomPool ) -> str | None: - """Return the recipe label that reproduces *path* byte for byte.""" + """Return the recipe label that reproduces *path* byte for byte. + + Equal bytes need equal (name, crc32) members, which the archive's + central directory gives without inflating anything: building every + recipe at DEFLATE level 9 first took 94 s and 1 GB at pipeline step 1c. + """ + held = _member_list(path) + if held is None: + return None original = path.read_bytes() for label, recipe in sorted(recipes.items()): + if sorted((name, crc.lower()) for name, crc in recipe) != held: + continue blob = pool.build(recipe) if blob is not None and blob == original: return label return None +def _member_list(path: Path) -> list[tuple[str, str]] | None: + """Sorted (name, crc32) of an archive's members, from its directory.""" + try: + with zipfile.ZipFile(path) as archive: + return sorted( + (info.filename, format(info.CRC, "08x")) + for info in archive.infolist() if not info.is_dir() + ) + except (zipfile.BadZipFile, OSError) as exc: + print(f" UNREADABLE: {path}: {exc}", file=sys.stderr) + return None + + def reconstruct( target_md5: str, recipes: dict[str, Recipe], pool: AtomPool ) -> tuple[str, bytes] | None: diff --git a/tests/test_romset_recipes.py b/tests/test_romset_recipes.py index 8e9baf11..6ec5d4ec 100644 --- a/tests/test_romset_recipes.py +++ b/tests/test_romset_recipes.py @@ -8,6 +8,7 @@ import sys import tempfile import unittest import zipfile +import zlib from pathlib import Path ROOT = Path(__file__).resolve().parent.parent @@ -557,5 +558,35 @@ class TheMissingReportNamesItsTargets(unittest.TestCase): self.assertIn("NO RECIPE: scummclassic.zip md5=dddddddd", text) self.assertLess(text.index("neogeo.zip"), text.index("aes.zip")) + +class IdentificationReadsTheDirectoryFirst(unittest.TestCase): + """identify_archive built every recipe at DEFLATE level 9 before + comparing bytes: 94 s and 1 GB at pipeline step 1c, mostly for recipes + whose members could never match.""" + + def test_a_recipe_with_other_members_is_never_built(self): + from torrentzip import build_torrentzip # noqa: PLC0415 + + import romset_recipes # noqa: PLC0415 + + rom = b"rom bytes" + crc = format(zlib.crc32(rom), "08x") + with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as tmp: + archive = Path(tmp) / "set.zip" + archive.write_bytes(build_torrentzip([("a.rom", rom)])) + + class Pool: + built: list = [] + + def build(self, recipe): + self.built.append(recipe) + return build_torrentzip([(name, rom) for name, _crc in recipe]) + + pool = Pool() + label = romset_recipes.identify_archive( + archive, {"other": [("b.rom", crc)], "match": [("a.rom", crc)]}, pool) + self.assertEqual(label, "match") + self.assertEqual(pool.built, [[("a.rom", crc)]]) + if __name__ == "__main__": unittest.main()