mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-10 21:43:23 -05:00
fix: pin the digest order in cached entries
The cached hashes were rebuilt by iterating a set, so their order in each database entry followed set hashing rather than a declared one. A run with a warm cache rewrote all 7,850 entries with no content change. The order is now the one compute_hashes returns, and a test holds a warm-cache run byte-identical to a --force rehash.
This commit is contained in:
1 parent
f9d1b43ccf
commit
0d59979f2a
2 files changed
+29
-10
No files matched your search
@@ -35,9 +35,11 @@ DEFAULT_OUTPUT = "database.json"
|
||||
|
||||
SKIP_PATTERNS = {".git", ".github", "__pycache__", ".cache", ".DS_Store", "desktop.ini"}
|
||||
|
||||
# Every digest database.json publishes per file. The schema requires all five,
|
||||
# so a cache entry missing any of them cannot serve a database entry.
|
||||
CACHED_HASHES = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"})
|
||||
# Every digest database.json publishes per file, in the order compute_hashes
|
||||
# returns them. The schema requires all five, so a cache entry missing any of
|
||||
# them cannot serve a database entry, and the order is fixed here so a warm
|
||||
# cache serialises an entry exactly like a fresh hash does.
|
||||
CACHED_HASHES = ("sha1", "md5", "sha256", "crc32", "adler32")
|
||||
|
||||
|
||||
def should_skip(path: Path) -> bool:
|
||||
@@ -91,7 +93,7 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
|
||||
# without it, so the loss survived every later run. Requiring the
|
||||
# full set instead turns a partial cache entry into a miss, which
|
||||
# heals it.
|
||||
if fresh and CACHED_HASHES.issubset(cached):
|
||||
if fresh and all(name in cached for name in CACHED_HASHES):
|
||||
hashes = {name: cached[name] for name in CACHED_HASHES}
|
||||
sha1 = hashes["sha1"]
|
||||
is_variant = "/.variants/" in rel_path or "\\.variants\\" in rel_path
|
||||
|
||||
@@ -181,21 +181,38 @@ class HashCacheKeepsEveryDigest(unittest.TestCase):
|
||||
def test_cache_hit_serves_the_full_digest_set(self):
|
||||
files, _, cache = self.generate_db.scan_bios_dir(self.bios, {}, force=False)
|
||||
entry = next(iter(files.values()))
|
||||
self.assertTrue(self.generate_db.CACHED_HASHES.issubset(entry))
|
||||
self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(entry))
|
||||
|
||||
# Second pass, this time served entirely from the cache.
|
||||
again, _, cache2 = self.generate_db.scan_bios_dir(self.bios, cache, force=False)
|
||||
entry2 = next(iter(again.values()))
|
||||
self.assertTrue(
|
||||
self.generate_db.CACHED_HASHES.issubset(entry2),
|
||||
f"cache hit lost {self.generate_db.CACHED_HASHES - set(entry2)}",
|
||||
set(self.generate_db.CACHED_HASHES).issubset(entry2),
|
||||
f"cache hit lost {set(self.generate_db.CACHED_HASHES) - set(entry2)}",
|
||||
)
|
||||
self.assertEqual(
|
||||
{k: entry[k] for k in self.generate_db.CACHED_HASHES},
|
||||
{k: entry2[k] for k in self.generate_db.CACHED_HASHES},
|
||||
)
|
||||
self.assertTrue(
|
||||
self.generate_db.CACHED_HASHES.issubset(next(iter(cache2.values())))
|
||||
set(self.generate_db.CACHED_HASHES).issubset(next(iter(cache2.values())))
|
||||
)
|
||||
|
||||
def test_a_warm_cache_serialises_exactly_like_a_fresh_hash(self):
|
||||
"""Key order must not depend on whether the cache was warm.
|
||||
|
||||
Rebuilding the dict by iterating a set made the order follow set
|
||||
hashing, so a run with a warm cache rewrote all 7,850 entries with
|
||||
their digests in a different order and no content change.
|
||||
"""
|
||||
fresh, _, cache = self.generate_db.scan_bios_dir(self.bios, {}, force=True)
|
||||
warm, _, _ = self.generate_db.scan_bios_dir(self.bios, cache, force=False)
|
||||
self.assertEqual(
|
||||
[list(entry) for entry in fresh.values()],
|
||||
[list(entry) for entry in warm.values()],
|
||||
)
|
||||
self.assertEqual(
|
||||
json.dumps(fresh, indent=2), json.dumps(warm, indent=2)
|
||||
)
|
||||
|
||||
def test_a_partial_cache_entry_is_rehashed_instead_of_trusted(self):
|
||||
@@ -204,8 +221,8 @@ class HashCacheKeepsEveryDigest(unittest.TestCase):
|
||||
cache[key].pop("adler32")
|
||||
files, _, healed = self.generate_db.scan_bios_dir(self.bios, cache, force=False)
|
||||
entry = next(iter(files.values()))
|
||||
self.assertTrue(self.generate_db.CACHED_HASHES.issubset(entry))
|
||||
self.assertTrue(self.generate_db.CACHED_HASHES.issubset(healed[key]))
|
||||
self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(entry))
|
||||
self.assertTrue(set(self.generate_db.CACHED_HASHES).issubset(healed[key]))
|
||||
|
||||
|
||||
class PreservedLargeFileEntries(unittest.TestCase):
|
||||
|
||||
Reference in new issue
Block a user