From eac2eaec8307eb9e99e8a851cb1707c6b29a37f7 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:36:47 +0200 Subject: [PATCH] fix: count only the hash changes an export writes RomM kept hashes the truth contradicts beside the corrected one, RetroDECK kept a cleared md5, Recalbox counted changes on an element it writes back whole, and two record formats counted fills no record could hold. --- scripts/export_native.py | 4 +- scripts/exporter/base_exporter.py | 14 ++++++ scripts/exporter/recalbox_exporter.py | 11 +++++ scripts/exporter/retrodeck_exporter.py | 6 +++ scripts/exporter/romm_exporter.py | 16 ++++++- scripts/exporter/systemdat_exporter.py | 3 ++ tests/test_export_counts.py | 65 ++++++++++++++++++++++++++ 7 files changed, 116 insertions(+), 3 deletions(-) diff --git a/scripts/export_native.py b/scripts/export_native.py index 5a623b67..92604aed 100644 --- a/scripts/export_native.py +++ b/scripts/export_native.py @@ -371,7 +371,7 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges: changes.filled.extend( f"{label} {field_name}" for field_name in entry.filled - if exporter.states(entry, field_name) + if exporter.fills(entry, field_name) ) if entry.platform is None: continue @@ -389,7 +389,7 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges: for field_name in HASH_FIELDS if _hash_values(entry.platform, field_name) and not entry.hashes(field_name) - and exporter.states(entry, field_name) + and exporter.clears(entry, field_name) ) # A value the render rewrote though the model reads it as the # platform's own (RomM's sha1 with a leading space). diff --git a/scripts/exporter/base_exporter.py b/scripts/exporter/base_exporter.py index cf7753fe..34bdf9c2 100644 --- a/scripts/exporter/base_exporter.py +++ b/scripts/exporter/base_exporter.py @@ -179,6 +179,20 @@ class BaseExporter(ABC): """ return field_name in self.carries() + def fills(self, fe: NativeFile, field_name: str) -> bool: + """Whether a hash only the truth names lands in the written entry.""" + return self.states(fe, field_name) + + def clears(self, fe: NativeFile, field_name: str) -> bool: + """Whether a platform hash the truth contradicts leaves the entry.""" + return self.states(fe, field_name) + + @staticmethod + def record_fills(fe: NativeFile, field_name: str) -> bool: + """A record format writes a filled hash only when record_hash names + one: two values the truth accepts make no single record.""" + return bool(fe.record_hash(field_name)) + def withheld(self) -> list[str]: """Corrections the render proved and still did not write, named. diff --git a/scripts/exporter/recalbox_exporter.py b/scripts/exporter/recalbox_exporter.py index 95b01525..edb2550e 100644 --- a/scripts/exporter/recalbox_exporter.py +++ b/scripts/exporter/recalbox_exporter.py @@ -98,6 +98,17 @@ class Exporter(BaseExporter): return f" " + def states(self, fe: NativeFile, field_name: str) -> bool: + """An element it cannot rewrite is written back as Recalbox wrote it. + + es_bios.xsd makes md5 required: an entry whose md5 the truth + contradicts without naming another keeps its element whole, and + counting the md5 cleared announced a change absent from the file. + """ + if fe.platform is not None and not self.writable(fe): + return False + return super().states(fe, field_name) + @classmethod def writable(cls, fe: NativeFile, require: str = "") -> bool: """es_bios.xsd makes md5 and core required; without them, no element. diff --git a/scripts/exporter/retrodeck_exporter.py b/scripts/exporter/retrodeck_exporter.py index a3bffb53..5af216b6 100644 --- a/scripts/exporter/retrodeck_exporter.py +++ b/scripts/exporter/retrodeck_exporter.py @@ -247,6 +247,12 @@ class Exporter(BaseExporter): (field, value) for field, value in by_key[keys[0]].items() if field in cls.carries() and not (field == "system" and declared) ) + # A hash of theirs that ours no longer carries is one the truth + # contradicts: kept, it went on rejecting the dump the emulator + # loads while the summary counted it cleared. + for field in ("md5", "sha256"): + if field in cls.carries() and field not in by_key[keys[0]]: + combined.pop(field, None) if _md5_set(entry.get("md5")) == _md5_set(combined.get("md5")): combined["md5"] = entry.get("md5") elif isinstance(entry.get("md5"), list) and isinstance(combined.get("md5"), str): diff --git a/scripts/exporter/romm_exporter.py b/scripts/exporter/romm_exporter.py index 33292bfa..8c37f5e8 100644 --- a/scripts/exporter/romm_exporter.py +++ b/scripts/exporter/romm_exporter.py @@ -18,7 +18,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from scraper.romm_scraper import SLUG_MAP from .base_exporter import BaseExporter -from .baseline import NativeFile, NativeSystem, Report +from .baseline import NativeFile, NativeSystem, Report, _hash_values SOURCE_URL = ( "https://raw.githubusercontent.com/rommapp/romm/master/backend/models" @@ -143,11 +143,25 @@ class Exporter(BaseExporter): value = fe.record_hash(hash_name) if value: self._write(entry, field, value, fe, hash_name) + elif self._contradicted(fe, hash_name): + # RomM accepts any one hash: a value the truth + # rejects, left beside the corrected md5, still + # passes the dump it describes. + entry.pop(field, None) output[key] = entry text = json.dumps(output, indent=2, ensure_ascii=False) + "\n" return {self.native_filename(): text} + @staticmethod + def _contradicted(fe: NativeFile, name: str) -> bool: + """A hash the platform states and the truth, taking the entry over, + does not keep.""" + return bool(_hash_values(fe.platform or {}, name)) and not fe.hashes(name) + + def fills(self, fe: NativeFile, field_name: str) -> bool: + return self.states(fe, field_name) and self.record_fills(fe, field_name) + def _write( self, entry: dict, field: str, value: str, fe: NativeFile, name: str ) -> None: diff --git a/scripts/exporter/systemdat_exporter.py b/scripts/exporter/systemdat_exporter.py index fdae2e3c..da795380 100644 --- a/scripts/exporter/systemdat_exporter.py +++ b/scripts/exporter/systemdat_exporter.py @@ -134,6 +134,9 @@ class Exporter(BaseExporter): ) return lines + def fills(self, fe: NativeFile, field_name: str) -> bool: + return self.states(fe, field_name) and self.record_fills(fe, field_name) + def render( self, systems: dict[str, NativeSystem], diff --git a/tests/test_export_counts.py b/tests/test_export_counts.py index 5d6da8f2..562ed668 100644 --- a/tests/test_export_counts.py +++ b/tests/test_export_counts.py @@ -786,5 +786,70 @@ class ACorruptOriginalIsRefused(unittest.TestCase): {exporter.native_filename(): '{"nes": {'}) +class TheSummaryCountsWhatTheFileCarries(unittest.TestCase): + """RomM kept the sha1 and crc the truth contradicts beside the corrected + md5, and RomM accepts any one hash; RetroDECK kept a cleared md5 and + Recalbox an element it could not rewrite, both counted cleared; RomM and + System.dat counted a sha1 filled that no record could hold.""" + + def _contradicted(self): + return NativeFile( + "scph5501.bin", "scph5501.bin", "psx", + platform={"size": 524288, "crc32": "deadbeef", "md5": A, "sha1": "1" * 40}, + truth={"md5": B}, + ) + + def test_romm_drops_the_hashes_it_counts_cleared(self): + from export_native import landed_changes # noqa: PLC0415 + from exporter.baseline import Report # noqa: PLC0415 + + fe = self._contradicted() + systems = {"psx": NativeSystem("psx", files=[fe])} + original = json.dumps({"psx:scph5501.bin": { + "size": "524288", "crc": "deadbeef", "md5": A, "sha1": "1" * 40}}) + exporter = Romm() + written = json.loads(exporter.render( + systems, Report(), {exporter.native_filename(): original}, + )[exporter.native_filename()])["psx:scph5501.bin"] + self.assertEqual(written.get("md5"), B) + self.assertNotIn("sha1", written) + self.assertNotIn("crc", written) + cleared = landed_changes(exporter, systems).cleared + self.assertEqual(sorted(cleared), ["psx/scph5501.bin crc32", "psx/scph5501.bin sha1"]) + + def test_a_fill_no_record_holds_is_not_counted(self): + from export_native import landed_changes # noqa: PLC0415 + + fe = NativeFile( + "scph5501.bin", "scph5501.bin", "psx", + platform={"md5": A}, + truth={"md5": [A, B], "sha1": ["1" * 40, "2" * 40]}, + filled=["sha1"], + ) + systems = {"psx": NativeSystem("psx", files=[fe])} + for exporter in (Romm(), SystemDat()): + with self.subTest(exporter=type(exporter).__module__): + self.assertEqual(landed_changes(exporter, systems).filled, []) + + def test_retrodeck_drops_a_cleared_md5(self): + merged = RetroDeck._merge( + [{"filename": "scph5501.bin", "system": "psx", "md5": A}], + [OrderedDict(filename="scph5501.bin", system="psx", + sha256="e" * 64)], + ) + self.assertNotIn("md5", merged[0]) + + def test_recalbox_counts_nothing_on_an_element_it_keeps(self): + from export_native import landed_changes # noqa: PLC0415 + + fe = NativeFile( + "scph5501.bin", "scph5501.bin", "psx", + platform={"md5": A, "size": 1, "core": "libretro/pcsx_rearmed"}, + truth={"sha1": "1" * 40, "size": 2}, + ) + changes = landed_changes(Recalbox(), {"psx": NativeSystem("psx", files=[fe])}) + self.assertEqual(changes.cleared, []) + + if __name__ == "__main__": unittest.main()