fix: keep a mister entry whole without a size

Downloader rejects an entry without an integer size (db_entity.py:187)
and compares it after each fetch (fetch_file_worker.py:127). A new hash
the truth gives no size for is withheld and named, not written beside
the old size nor reported as a size removal.
This commit is contained in:
Abdessamad Derraz committed 2026-10-21 05:12:02 +02:00
1 parent 2e29b3be03
commit d48e81861c
2 files changed
+47 -2

No files matched your search

+22 -2
View File
@@ -113,6 +113,7 @@ class Exporter(BaseExporter):
database = json.loads(original, object_pairs_hook=OrderedDict)
indexed = self._by_path(systems)
self._written = set()
self._unsized: list[str] = []
for path, entry in database.get("files", {}).items():
fe = indexed.get(path)
@@ -124,9 +125,17 @@ class Exporter(BaseExporter):
# entry without one, a new size beside the old hash describes
# no file, and the entry is left as MiSTer wrote it.
continue
size = fe.size()
if md5 != entry.get("hash") and not size:
# Downloader refuses an entry without an integer size
# (db_entity.py:187) and compares it after each fetch
# (fetch_file_worker.py:127). The old size beside the new
# hash describes the dump the truth rejects, so the entry
# stays as MiSTer wrote it.
self._unsized.append(path)
continue
self._written.add(id(fe))
entry["hash"] = md5
size = fe.size()
if size:
entry["size"] = size
@@ -143,6 +152,14 @@ class Exporter(BaseExporter):
produced[_ARCHIVE] = archive if unchanged else rebuild_archive(archive, _DB_NAME, text)
return produced
def withheld(self) -> list[str]:
"""Each hash correction left out for want of the new dump's size."""
return [
f"{path} keeps its hash: the truth states no size for the new "
"dump, and the Downloader checks every entry by size"
for path in getattr(self, "_unsized", [])
]
def validate(
self,
systems: dict[str, NativeSystem],
@@ -169,6 +186,9 @@ class Exporter(BaseExporter):
for path, fe in indexed.items():
declared = files.get(path)
md5 = fe.hash("md5")
if declared is not None and md5 and declared.get("hash") != md5:
if (
declared is not None and md5 and declared.get("hash") != md5
and path not in getattr(self, "_unsized", [])
):
issues.append(f"hash not applied: {path}")
return issues
+25
View File
@@ -1670,6 +1670,31 @@ class MisterDatabaseComesBackAsWritten(unittest.TestCase):
export_native._write_produced(Path(tmp), {"a.zip": b"PK\x03\x04\xff"}, {})
self.assertEqual((Path(tmp) / "a.zip").read_bytes(), b"PK\x03\x04\xff")
def test_a_new_hash_without_a_size_leaves_the_entry_whole(self):
"""The Downloader checks each entry by size (fetch_file_worker.py:127):
the old size beside the new hash described no file, and the report
announced a size removal the database did not carry."""
from export_native import landed_changes # noqa: PLC0415
database = {"db_id": "bios_db", "files": {
"games/X/boot.rom": {"hash": "a" * 32, "size": 100, "url": "u"},
}}
original = json.dumps(database)
scraped = {"systems": {"x": {"files": [{
"name": "boot.rom", "destination": "X/boot.rom",
"md5": "a" * 32, "size": 100,
}]}}}
truth = {"systems": {"x": {"files": [{
"name": "boot.rom", "path": "X/boot.rom", "md5": "b" * 32, "_cores": ["x"],
}]}}}
exporter = discover_exporters()["misterfpga"]()
systems, report = build_native_model(truth, scraped)
produced = exporter.render(systems, report, {"bios_db.json": original}, scraped)
self.assertEqual(produced["bios_db.json"], original)
self.assertEqual(landed_changes(exporter, systems).records, [])
self.assertEqual(len(exporter.withheld()), 1)
self.assertEqual(exporter.validate(systems, produced), [])
def test_an_empty_truth_hands_back_the_database(self):
archive = UPSTREAM_CACHE / "misterfpga" / "bios_db.json.zip"
if not archive.exists():