diff --git a/scripts/exporter/romm_exporter.py b/scripts/exporter/romm_exporter.py index fcdef4a2..f5df5077 100644 --- a/scripts/exporter/romm_exporter.py +++ b/scripts/exporter/romm_exporter.py @@ -52,12 +52,6 @@ class Exporter(BaseExporter): def native_sources() -> dict[str, str]: return {"known_bios_files.json": SOURCE_URL} - def __init__(self) -> None: - super().__init__() - # Filled by render; empty until then, so writable() answers before it. - self._platform_keys: set[tuple[str, str]] = set() - self._first_addition: dict[tuple[str, str], int] = {} - @staticmethod def _verifiable(fe: NativeFile) -> bool: return bool(fe.size()) and any( @@ -102,7 +96,15 @@ class Exporter(BaseExporter): originals: dict[str, str], scraped: dict | None = None, ) -> dict[str, str]: - output: OrderedDict[str, dict] = OrderedDict() + # RomM's own fixture is the base: its key order stays, and a key our + # transcription lacks (added after the scrape, or filtered from the + # model) is kept. Rebuilt from the model in our order, three hash + # corrections came out as a diff of the whole file. + original = originals.get(self.native_filename(), "") + output: OrderedDict[str, OrderedDict] = ( + json.loads(original, object_pairs_hook=OrderedDict) + if original else OrderedDict() + ) self._platform_keys = { (system.native_id, fe.name) for system in systems.values() @@ -125,7 +127,9 @@ class Exporter(BaseExporter): for fe in sorted(system.files, key=lambda f: f.name): if not self.writable(fe): continue - entry: OrderedDict[str, str] = OrderedDict() + key = f"{system.native_id}:{fe.name}" + # Their entry keeps its fields and their order; ours fill in. + entry: OrderedDict[str, str] = OrderedDict(output.get(key, {})) # The fixture states every value as a string, size included. # Where the model settles no size, RomM's own value stands; # str(None) wrote "None", and RomM's int() on it raised. @@ -134,16 +138,11 @@ class Exporter(BaseExporter): size = fe.platform.get("size") if size is not None: entry["size"] = str(int(size)) - crc = fe.record_hash("crc32") - if crc: - entry["crc"] = crc - md5 = fe.record_hash("md5") - if md5: - entry["md5"] = md5 - sha1 = fe.record_hash("sha1") - if sha1: - entry["sha1"] = sha1 - output[f"{system.native_id}:{fe.name}"] = entry + for field, hash_name in (("crc", "crc32"), ("md5", "md5"), ("sha1", "sha1")): + value = fe.record_hash(hash_name) + if value: + entry[field] = value + output[key] = entry text = json.dumps(output, indent=2, ensure_ascii=False) + "\n" return {self.native_filename(): text} diff --git a/tests/test_exporters.py b/tests/test_exporters.py index a19ab1ac..1039758d 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -14,6 +14,7 @@ and skip when it has not been populated (python scripts/export_native.py from __future__ import annotations +import difflib import json import os import re @@ -22,6 +23,7 @@ import sys import tempfile import unittest import xml.etree.ElementTree as ET +from collections import OrderedDict from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent @@ -782,6 +784,46 @@ class RommExport(unittest.TestCase): self.assertTrue(all(k.startswith("psx:") for k in data)) +class RommKeepsItsFixture(unittest.TestCase): + """The export rebuilt known_bios_files.json from our model in our order: + 3 hash corrections came out as a 3116-line diff, and a key RomM had that + our transcription lacked was deleted.""" + + def test_order_unknown_keys_and_untouched_entries_survive(self): + theirs = OrderedDict([ + ("psx:zzz_first.bin", OrderedDict([("size", "1024"), ("md5", "a" * 32)])), + ("psx:only_theirs.bin", OrderedDict([("size", "16"), ("crc", "deadbeef")])), + ("psx:scph5501.bin", OrderedDict([("size", "524288"), ("md5", "b" * 32)])), + ]) + original = json.dumps(theirs, indent=2) + "\n" + scraped = {"systems": {"sony-playstation": {"native_id": "psx", "files": [ + {"name": "zzz_first.bin", "destination": "psx/zzz_first.bin", + "native_system": "psx", "md5": "a" * 32, "size": 1024}, + {"name": "scph5501.bin", "destination": "psx/scph5501.bin", + "native_system": "psx", "md5": "b" * 32, "size": 524288}, + ]}}} + truth = {"systems": {"sony-playstation": {"files": [{ + "name": "scph5501.bin", "path": "scph5501.bin", "md5": "c" * 32, + "size": 524288, "required": True, "_cores": ["beetle_psx"], + }]}}} + systems, report = build_native_model(truth, scraped) + produced = discover_exporters()["romm"]().render( + systems, report, {"known_bios_files.json": original}, None + )["known_bios_files.json"] + data = json.loads(produced, object_pairs_hook=OrderedDict) + self.assertEqual( + list(data), ["psx:zzz_first.bin", "psx:only_theirs.bin", "psx:scph5501.bin"] + ) + self.assertEqual(data["psx:only_theirs.bin"], theirs["psx:only_theirs.bin"]) + self.assertEqual(data["psx:scph5501.bin"]["md5"], "c" * 32) + changed = [ + line for line in difflib.unified_diff( + original.splitlines(), produced.splitlines(), lineterm="", n=0 + ) if line[:1] in "+-" and line[:3] not in ("+++", "---") + ] + self.assertEqual(len(changed), 2, changed) + + class SystemDatKeepsSpaceIndentedLines(unittest.TestCase): """libretro indents 39 rom lines with four spaces. Read with a tab only, they left the base and came back last in their section, re-indented."""