diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 78b56cb2..0990af11 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -3175,6 +3175,11 @@ def _db_sha1_by_path(db: dict) -> dict[str, str]: return index +# The hashes a manifest entry records, and each file's once verified. +_MANIFEST_HASHES = frozenset({"sha1", "sha256"}) +_STABLE_HASHES: dict[tuple, dict] = {} + + def _stable_hashes(local_path: str, db: dict) -> dict: """The file's hashes, held against the scan that built the database. @@ -3184,12 +3189,20 @@ def _stable_hashes(local_path: str, db: dict) -> dict: database scanned the same path, and a manifest that disagrees with it is refused rather than published. """ + stat = os.stat(local_path) + key = (os.path.normpath(local_path), stat.st_size, stat.st_mtime_ns, id(db)) + known = _STABLE_HASHES.get(key) + if known is not None: + # Verified against the scan once: twelve manifests name a file up + # to a dozen times, and rehashing it each time spent 163 s of 218. + return known expected = _db_sha1_by_path(db).get(os.path.normpath(local_path)) - hashes = compute_hashes(local_path) + hashes = compute_hashes(local_path, _MANIFEST_HASHES) for _attempt in range(2): if expected is None or hashes["sha1"] == expected: + _STABLE_HASHES[key] = hashes return hashes - hashes = compute_hashes(local_path) + hashes = compute_hashes(local_path, _MANIFEST_HASHES) raise RuntimeError( f"{local_path}: sha1 {hashes['sha1']} contradicts the database scan " f"({expected}); rehash the file or regenerate database.json" diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index f59627d2..7d325d6f 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -3643,5 +3643,33 @@ class EmulatorPagesStateTheirDataForMachines(unittest.TestCase): self.assertNotIn("identifier", data["hasPart"][1]) + +class AManifestHashesAFileOnce(unittest.TestCase): + """Every manifest entry rehashed its file with five algorithms: 70 476 + calls for 15 011 files, 163 s of a 218 s step.""" + + def test_verified_hashes_are_reused(self): + from scripts import generate_pack + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp: + path = Path(tmp, "a.bin") + path.write_bytes(b"abc") + sha1 = hashlib.sha1(b"abc").hexdigest() + db = {"files": {sha1: {"path": str(path)}}} + calls = [] + real = generate_pack.compute_hashes + + def counting(file_path, algorithms=None): + calls.append(algorithms) + return real(file_path, algorithms) + + with mock.patch.object(generate_pack, "compute_hashes", counting): + first = generate_pack._stable_hashes(str(path), db) + second = generate_pack._stable_hashes(str(path), db) + self.assertEqual(first, second) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0], frozenset({"sha1", "sha256"})) + + if __name__ == "__main__": unittest.main()