diff --git a/scripts/exporter/misterfpga_exporter.py b/scripts/exporter/misterfpga_exporter.py index 0d0df758..79de82d7 100644 --- a/scripts/exporter/misterfpga_exporter.py +++ b/scripts/exporter/misterfpga_exporter.py @@ -113,6 +113,7 @@ class Exporter(BaseExporter): database = json.loads(original, object_pairs_hook=OrderedDict) indexed = self._by_path(systems) self._written = set() + self._unsized: list[str] = [] for path, entry in database.get("files", {}).items(): fe = indexed.get(path) @@ -124,9 +125,17 @@ class Exporter(BaseExporter): # entry without one, a new size beside the old hash describes # no file, and the entry is left as MiSTer wrote it. continue + size = fe.size() + if md5 != entry.get("hash") and not size: + # Downloader refuses an entry without an integer size + # (db_entity.py:187) and compares it after each fetch + # (fetch_file_worker.py:127). The old size beside the new + # hash describes the dump the truth rejects, so the entry + # stays as MiSTer wrote it. + self._unsized.append(path) + continue self._written.add(id(fe)) entry["hash"] = md5 - size = fe.size() if size: entry["size"] = size @@ -143,6 +152,14 @@ class Exporter(BaseExporter): produced[_ARCHIVE] = archive if unchanged else rebuild_archive(archive, _DB_NAME, text) return produced + def withheld(self) -> list[str]: + """Each hash correction left out for want of the new dump's size.""" + return [ + f"{path} keeps its hash: the truth states no size for the new " + "dump, and the Downloader checks every entry by size" + for path in getattr(self, "_unsized", []) + ] + def validate( self, systems: dict[str, NativeSystem], @@ -169,6 +186,9 @@ class Exporter(BaseExporter): for path, fe in indexed.items(): declared = files.get(path) md5 = fe.hash("md5") - if declared is not None and md5 and declared.get("hash") != md5: + if ( + declared is not None and md5 and declared.get("hash") != md5 + and path not in getattr(self, "_unsized", []) + ): issues.append(f"hash not applied: {path}") return issues diff --git a/tests/test_exporters.py b/tests/test_exporters.py index c2f4d91c..fc3abc00 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1670,6 +1670,31 @@ class MisterDatabaseComesBackAsWritten(unittest.TestCase): export_native._write_produced(Path(tmp), {"a.zip": b"PK\x03\x04\xff"}, {}) self.assertEqual((Path(tmp) / "a.zip").read_bytes(), b"PK\x03\x04\xff") + def test_a_new_hash_without_a_size_leaves_the_entry_whole(self): + """The Downloader checks each entry by size (fetch_file_worker.py:127): + the old size beside the new hash described no file, and the report + announced a size removal the database did not carry.""" + from export_native import landed_changes # noqa: PLC0415 + + database = {"db_id": "bios_db", "files": { + "games/X/boot.rom": {"hash": "a" * 32, "size": 100, "url": "u"}, + }} + original = json.dumps(database) + scraped = {"systems": {"x": {"files": [{ + "name": "boot.rom", "destination": "X/boot.rom", + "md5": "a" * 32, "size": 100, + }]}}} + truth = {"systems": {"x": {"files": [{ + "name": "boot.rom", "path": "X/boot.rom", "md5": "b" * 32, "_cores": ["x"], + }]}}} + exporter = discover_exporters()["misterfpga"]() + systems, report = build_native_model(truth, scraped) + produced = exporter.render(systems, report, {"bios_db.json": original}, scraped) + self.assertEqual(produced["bios_db.json"], original) + self.assertEqual(landed_changes(exporter, systems).records, []) + self.assertEqual(len(exporter.withheld()), 1) + self.assertEqual(exporter.validate(systems, produced), []) + def test_an_empty_truth_hands_back_the_database(self): archive = UPSTREAM_CACHE / "misterfpga" / "bios_db.json.zip" if not archive.exists():