diff --git a/exports/misterfpga.json b/exports/misterfpga.json index 0aa1df62..ea1da06f 100644 --- a/exports/misterfpga.json +++ b/exports/misterfpga.json @@ -4,7 +4,8 @@ "bios_db.json.zip": "https://raw.githubusercontent.com/ajgowans/BiosDB_MiSTer/7cfa7a3e84af078d0efba74fa46c50357c4f567c/bios_db.json.zip" }, "files": [ - "bios_db.json" + "bios_db.json", + "bios_db.json.zip" ], "summary": "81 kept, 0 added, 0 hashes corrected, 0 requirements corrected", "counts": { diff --git a/exports/misterfpga/bios_db.json.zip b/exports/misterfpga/bios_db.json.zip new file mode 100644 index 00000000..bd0b97e0 Binary files /dev/null and b/exports/misterfpga/bios_db.json.zip differ diff --git a/scripts/export_native.py b/scripts/export_native.py index 98ea7454..629eb03f 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -339,6 +339,9 @@ def collect_originals( continue if archive: originals.update(_read_archive(unpack, payload)) + keep = getattr(exporter, "keep_archive", None) + if callable(keep): + keep(relative, payload) else: originals[relative] = payload.decode("utf-8", errors="replace") # One platform's unreadable file fails that platform: raised, it @@ -760,7 +763,9 @@ def export_platform( return True, messages -def _write_produced(root: Path, produced: dict[str, str], modes: dict[str, int]) -> None: +def _write_produced( + root: Path, produced: dict[str, str | bytes], modes: dict[str, int] +) -> None: """Write each file as its bytes, whole or not at all, with its mode. write_text translated line ends on Windows, so FirmwareDatabase.cs's @@ -771,7 +776,8 @@ def _write_produced(root: Path, produced: dict[str, str], modes: dict[str, int]) for relative, content in produced.items(): path = root / relative path.parent.mkdir(parents=True, exist_ok=True) - write_bytes_atomic(str(path), content.encode("utf-8")) + data = content if isinstance(content, bytes) else content.encode("utf-8") + write_bytes_atomic(str(path), data) path.chmod(modes.get(relative, 0o644)) diff --git a/scripts/exporter/base_exporter.py b/scripts/exporter/base_exporter.py index 31056015..b91289a0 100644 --- a/scripts/exporter/base_exporter.py +++ b/scripts/exporter/base_exporter.py @@ -126,6 +126,15 @@ class BaseExporter(ABC): """ return True + def keep_archive(self, relative: str, payload: bytes) -> None: + """Hold the bytes of an original archive the render may hand back.""" + if not hasattr(self, "_archives"): + self._archives: dict[str, bytes] = {} + self._archives[relative] = payload + + def archive(self, relative: str) -> bytes | None: + return getattr(self, "_archives", {}).get(relative) + @staticmethod def file_modes() -> dict[str, int]: """The mode of each file the platform publishes other than 0644.""" diff --git a/scripts/exporter/misterfpga_exporter.py b/scripts/exporter/misterfpga_exporter.py index 7869a336..0d0df758 100644 --- a/scripts/exporter/misterfpga_exporter.py +++ b/scripts/exporter/misterfpga_exporter.py @@ -21,6 +21,27 @@ SOURCE_URL = ( _DB_NAME = "bios_db.json" +_ARCHIVE = "bios_db.json.zip" + + +def rebuild_archive(original: bytes, member: str, text: str) -> bytes: + """The archive with one member's content replaced, as MiSTer writes it. + + The member keeps its name, date, compression and attributes, so the + archive differs from MiSTer's only by the content that changed. + """ + with zipfile.ZipFile(io.BytesIO(original)) as source: + info = source.getinfo(member) + rebuilt = zipfile.ZipInfo(info.filename, date_time=info.date_time) + rebuilt.compress_type = info.compress_type + rebuilt.external_attr = info.external_attr + rebuilt.create_system = info.create_system + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as target: + target.writestr(rebuilt, text.encode("utf-8")) + return buffer.getvalue() + + class Exporter(BaseExporter): """Write MiSTer's bios_db.json, corrected.""" @@ -82,7 +103,7 @@ class Exporter(BaseExporter): report: Report, originals: dict[str, str], scraped: dict | None = None, - ) -> dict[str, str]: + ) -> dict[str, str | bytes]: original = originals.get(_DB_NAME) or originals.get("bios_db.json.zip") if not original: raise ValueError( @@ -111,7 +132,16 @@ class Exporter(BaseExporter): # MiSTer publishes the database as one line, without a final # newline: written with an indent it came out as 1499 changed lines. - return {_DB_NAME: json.dumps(database, ensure_ascii=False)} + text = json.dumps(database, ensure_ascii=False) + produced: dict[str, str | bytes] = {_DB_NAME: text} + # MiSTer's Downloader fetches the archive, not the JSON in it: handed + # back as it came when nothing changed, rebuilt around the new + # member otherwise. + archive = self.archive(_ARCHIVE) + if archive is not None: + unchanged = text == originals.get(_DB_NAME) + produced[_ARCHIVE] = archive if unchanged else rebuild_archive(archive, _DB_NAME, text) + return produced def validate( self, diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 47585e76..402b2e66 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1546,6 +1546,45 @@ class MisterDatabaseComesBackAsWritten(unittest.TestCase): """The BiosDB is one line of JSON. Written with indent=2 it came out as 1499 changed lines for a database whose content had not changed.""" + def _render(self, truth: dict): + archive = UPSTREAM_CACHE / "misterfpga" / "bios_db.json.zip" + if not archive.exists(): + self.skipTest("upstream cache not populated") + raw = archive.read_bytes() + exporter = discover_exporters()["misterfpga"]() + exporter.keep_archive("bios_db.json.zip", raw) + originals = exporter.unpack(raw) + config = native_layer(load_platform_config("misterfpga", str(REPO_ROOT / "platforms"))) + systems, report = build_native_model(truth, config) + return raw, exporter.render(systems, report, originals, config) + + def test_the_archive_mister_fetches_is_published_as_it_came(self): + """The Downloader fetches bios_db.json.zip, and the export gave only + the JSON inside it.""" + raw, produced = self._render({"systems": {}}) + self.assertEqual(produced["bios_db.json.zip"], raw) + + def test_a_rebuilt_archive_keeps_the_member_as_mister_writes_it(self): + raw, _ = self._render({"systems": {}}) + from exporter.misterfpga_exporter import rebuild_archive # noqa: PLC0415 + + rebuilt = rebuild_archive(raw, "bios_db.json", '{"changed": true}') + with zipfile.ZipFile(io.BytesIO(raw)) as before, \ + zipfile.ZipFile(io.BytesIO(rebuilt)) as after: + old, new = before.infolist()[0], after.infolist()[0] + self.assertEqual(after.read("bios_db.json"), b'{"changed": true}') + self.assertEqual( + (old.filename, old.date_time, old.compress_type, old.external_attr), + (new.filename, new.date_time, new.compress_type, new.external_attr), + ) + + def test_bytes_are_written_untouched(self): + import export_native # noqa: PLC0415 + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + export_native._write_produced(Path(tmp), {"a.zip": b"PK\x03\x04\xff"}, {}) + self.assertEqual((Path(tmp) / "a.zip").read_bytes(), b"PK\x03\x04\xff") + def test_an_empty_truth_hands_back_the_database(self): archive = UPSTREAM_CACHE / "misterfpga" / "bios_db.json.zip" if not archive.exists():