From 0d59979f2aef77ffe5c3c312c824095a0321d552 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 11 Aug 2026 01:46:46 +0200 Subject: [PATCH] fix: pin the digest order in cached entries The cached hashes were rebuilt by iterating a set, so their order in each database entry followed set hashing rather than a declared one. A run with a warm cache rewrote all 7,850 entries with no content change. The order is now the one compute_hashes returns, and a test holds a warm-cache run byte-identical to a --force rehash. --- scripts/generate_db.py | 10 ++++++---- tests/test_large_file_cache.py | 29 +++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/scripts/generate_db.py b/scripts/generate_db.py index 82923fc8..7bc61161 100644 --- a/scripts/generate_db.py +++ b/scripts/generate_db.py @@ -35,9 +35,11 @@ DEFAULT_OUTPUT = "database.json" SKIP_PATTERNS = {".git", ".github", "__pycache__", ".cache", ".DS_Store", "desktop.ini"} -# Every digest database.json publishes per file. The schema requires all five, -# so a cache entry missing any of them cannot serve a database entry. -CACHED_HASHES = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"}) +# Every digest database.json publishes per file, in the order compute_hashes +# returns them. The schema requires all five, so a cache entry missing any of +# them cannot serve a database entry, and the order is fixed here so a warm +# cache serialises an entry exactly like a fresh hash does. +CACHED_HASHES = ("sha1", "md5", "sha256", "crc32", "adler32") def should_skip(path: Path) -> bool: @@ -91,7 +93,7 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict, # without it, so the loss survived every later run. Requiring the # full set instead turns a partial cache entry into a miss, which # heals it. - if fresh and CACHED_HASHES.issubset(cached): + if fresh and all(name in cached for name in CACHED_HASHES): hashes = {name: cached[name] for name in CACHED_HASHES} sha1 = hashes["sha1"] is_variant = "/.variants/" in rel_path or "\\.variants\\" in rel_path diff --git a/tests/test_large_file_cache.py b/tests/test_large_file_cache.py index 5a765466..c0b1998c 100644 --- a/tests/test_large_file_cache.py +++ b/tests/test_large_file_cache.py @@ -181,21 +181,38 @@ class HashCacheKeepsEveryDigest(unittest.TestCase): def test_cache_hit_serves_the_full_digest_set(self): files, _, cache = self.generate_db.scan_bios_dir(self.bios, {}, force=False) entry = next(iter(files.values())) - self.assertTrue(self.generate_db.CACHED_HASHES.issubset(entry)) + self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(entry)) # Second pass, this time served entirely from the cache. again, _, cache2 = self.generate_db.scan_bios_dir(self.bios, cache, force=False) entry2 = next(iter(again.values())) self.assertTrue( - self.generate_db.CACHED_HASHES.issubset(entry2), - f"cache hit lost {self.generate_db.CACHED_HASHES - set(entry2)}", + set(self.generate_db.CACHED_HASHES).issubset(entry2), + f"cache hit lost {set(self.generate_db.CACHED_HASHES) - set(entry2)}", ) self.assertEqual( {k: entry[k] for k in self.generate_db.CACHED_HASHES}, {k: entry2[k] for k in self.generate_db.CACHED_HASHES}, ) self.assertTrue( - self.generate_db.CACHED_HASHES.issubset(next(iter(cache2.values()))) + set(self.generate_db.CACHED_HASHES).issubset(next(iter(cache2.values()))) + ) + + def test_a_warm_cache_serialises_exactly_like_a_fresh_hash(self): + """Key order must not depend on whether the cache was warm. + + Rebuilding the dict by iterating a set made the order follow set + hashing, so a run with a warm cache rewrote all 7,850 entries with + their digests in a different order and no content change. + """ + fresh, _, cache = self.generate_db.scan_bios_dir(self.bios, {}, force=True) + warm, _, _ = self.generate_db.scan_bios_dir(self.bios, cache, force=False) + self.assertEqual( + [list(entry) for entry in fresh.values()], + [list(entry) for entry in warm.values()], + ) + self.assertEqual( + json.dumps(fresh, indent=2), json.dumps(warm, indent=2) ) def test_a_partial_cache_entry_is_rehashed_instead_of_trusted(self): @@ -204,8 +221,8 @@ class HashCacheKeepsEveryDigest(unittest.TestCase): cache[key].pop("adler32") files, _, healed = self.generate_db.scan_bios_dir(self.bios, cache, force=False) entry = next(iter(files.values())) - self.assertTrue(self.generate_db.CACHED_HASHES.issubset(entry)) - self.assertTrue(self.generate_db.CACHED_HASHES.issubset(healed[key])) + self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(entry)) + self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(healed[key])) class PreservedLargeFileEntries(unittest.TestCase):