fix: pin the digest order in cached entries

The cached hashes were rebuilt by iterating a set, so their order in
each database entry followed set hashing rather than a declared one. A
run with a warm cache rewrote all 7,850 entries with no content change.
The order is now the one compute_hashes returns, and a test holds a
warm-cache run byte-identical to a --force rehash.
This commit is contained in:
Abdessamad Derraz committed 2026-08-11 01:46:46 +02:00
1 parent f9d1b43ccf
commit 0d59979f2a
2 files changed
+29 -10

No files matched your search

+6 -4
View File
@@ -35,9 +35,11 @@ DEFAULT_OUTPUT = "database.json"
SKIP_PATTERNS = {".git", ".github", "__pycache__", ".cache", ".DS_Store", "desktop.ini"}
# Every digest database.json publishes per file. The schema requires all five,
# so a cache entry missing any of them cannot serve a database entry.
CACHED_HASHES = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"})
# Every digest database.json publishes per file, in the order compute_hashes
# returns them. The schema requires all five, so a cache entry missing any of
# them cannot serve a database entry, and the order is fixed here so a warm
# cache serialises an entry exactly like a fresh hash does.
CACHED_HASHES = ("sha1", "md5", "sha256", "crc32", "adler32")
def should_skip(path: Path) -> bool:
@@ -91,7 +93,7 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
# without it, so the loss survived every later run. Requiring the
# full set instead turns a partial cache entry into a miss, which
# heals it.
if fresh and CACHED_HASHES.issubset(cached):
if fresh and all(name in cached for name in CACHED_HASHES):
hashes = {name: cached[name] for name in CACHED_HASHES}
sha1 = hashes["sha1"]
is_variant = "/.variants/" in rel_path or "\\.variants\\" in rel_path
+23 -6
View File
@@ -181,21 +181,38 @@ class HashCacheKeepsEveryDigest(unittest.TestCase):
def test_cache_hit_serves_the_full_digest_set(self):
files, _, cache = self.generate_db.scan_bios_dir(self.bios, {}, force=False)
entry = next(iter(files.values()))
self.assertTrue(self.generate_db.CACHED_HASHES.issubset(entry))
self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(entry))
# Second pass, this time served entirely from the cache.
again, _, cache2 = self.generate_db.scan_bios_dir(self.bios, cache, force=False)
entry2 = next(iter(again.values()))
self.assertTrue(
self.generate_db.CACHED_HASHES.issubset(entry2),
f"cache hit lost {self.generate_db.CACHED_HASHES - set(entry2)}",
set(self.generate_db.CACHED_HASHES).issubset(entry2),
f"cache hit lost {set(self.generate_db.CACHED_HASHES) - set(entry2)}",
)
self.assertEqual(
{k: entry[k] for k in self.generate_db.CACHED_HASHES},
{k: entry2[k] for k in self.generate_db.CACHED_HASHES},
)
self.assertTrue(
self.generate_db.CACHED_HASHES.issubset(next(iter(cache2.values())))
set(self.generate_db.CACHED_HASHES).issubset(next(iter(cache2.values())))
)
def test_a_warm_cache_serialises_exactly_like_a_fresh_hash(self):
"""Key order must not depend on whether the cache was warm.
Rebuilding the dict by iterating a set made the order follow set
hashing, so a run with a warm cache rewrote all 7,850 entries with
their digests in a different order and no content change.
"""
fresh, _, cache = self.generate_db.scan_bios_dir(self.bios, {}, force=True)
warm, _, _ = self.generate_db.scan_bios_dir(self.bios, cache, force=False)
self.assertEqual(
[list(entry) for entry in fresh.values()],
[list(entry) for entry in warm.values()],
)
self.assertEqual(
json.dumps(fresh, indent=2), json.dumps(warm, indent=2)
)
def test_a_partial_cache_entry_is_rehashed_instead_of_trusted(self):
@@ -204,8 +221,8 @@ class HashCacheKeepsEveryDigest(unittest.TestCase):
cache[key].pop("adler32")
files, _, healed = self.generate_db.scan_bios_dir(self.bios, cache, force=False)
entry = next(iter(files.values()))
self.assertTrue(self.generate_db.CACHED_HASHES.issubset(entry))
self.assertTrue(self.generate_db.CACHED_HASHES.issubset(healed[key]))
self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(entry))
self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(healed[key]))
class PreservedLargeFileEntries(unittest.TestCase):