From ff5e5145fdbf911d798a3036fe29c92900d898bb Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 16 Oct 2026 05:20:09 +0200 Subject: [PATCH] fix: keep recalbox and libretro entries on export --- scripts/exporter/recalbox_exporter.py | 54 +++++++++++++++++++---- scripts/exporter/systemdat_exporter.py | 54 ++++++++++++++++++----- tests/test_export_counts.py | 61 ++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 19 deletions(-) diff --git a/scripts/exporter/recalbox_exporter.py b/scripts/exporter/recalbox_exporter.py index b85ccf9f..9e5aa1fe 100644 --- a/scripts/exporter/recalbox_exporter.py +++ b/scripts/exporter/recalbox_exporter.py @@ -115,18 +115,56 @@ class Exporter(BaseExporter): '', ] + # Recalbox's own file is the base, so an element our transcription + # does not carry (added or renamed after a scrape) stays as written, + # and its system order with it. + theirs: list[tuple[str, str, list[dict[str, str]]]] = [] + original = originals.get(self.native_filename(), "") + if original: + try: + root = parse_untrusted_xml(original, self.native_filename()) + except (ParseError, ValueError): + root = None + if root is not None: + theirs = [ + (s.get("platform", ""), s.get("fullname", ""), + [dict(b.attrib) for b in s.iter("bios")]) + for s in root.iter("system") + ] - for system in sorted(systems.values(), key=lambda s: s.native_id): + ours: dict[str, tuple[NativeSystem, dict[str, str]]] = {} + for system in systems.values(): writable = [fe for fe in system.files if self.writable(fe)] - if not writable: - continue - fullname = quoteattr(self.display_name(system)) - platform = quoteattr(system.native_id) - lines.append(f" ") - for fe in writable: - lines.append(self._bios_element(fe, system.native_id)) + if writable: + ours[system.native_id] = (system, { + self._path(fe): self._bios_element(fe, system.native_id) + for fe in writable + }) + + def element(attrs: dict[str, str]) -> str: + rendered = " ".join(f"{k}={quoteattr(v)}" for k, v in attrs.items()) + return f" " + + def emit(platform: str, fullname: str, body: list[str]) -> None: + lines.append( + f" " + ) + lines.extend(body) lines.append(" ") + for platform, fullname, bios in theirs: + if platform not in ours: + emit(platform, fullname, [element(attrs) for attrs in bios]) + continue + system, elements = ours.pop(platform) + body = [elements.pop(attrs.get("path", ""), None) or element(attrs) + for attrs in bios] + body.extend(elements.values()) + emit(platform, self.display_name(system), body) + for platform in sorted(ours): + system, elements = ours[platform] + emit(platform, self.display_name(system), list(elements.values())) + lines.append("") lines.append("") return {self.native_filename(): "\n".join(lines)} diff --git a/scripts/exporter/systemdat_exporter.py b/scripts/exporter/systemdat_exporter.py index f60db66b..3b198490 100644 --- a/scripts/exporter/systemdat_exporter.py +++ b/scripts/exporter/systemdat_exporter.py @@ -36,6 +36,23 @@ def _quote(name: str, quoted: frozenset[str] = frozenset()) -> str: return f'"{name}"' if name in quoted or any(c in name for c in ' ()') else name +_ROM_NAME = re.compile(r'^\trom \( name ("[^"]+"|\S+)') + + +def _sections(original: str) -> list[tuple[str, list[str]]]: + """The system sections of a System.dat, each comment with its rom lines.""" + _, sep, body = original.partition("\ngame (") + if not sep: + return [] + sections: list[tuple[str, list[str]]] = [] + for line in body.splitlines(): + if line.startswith('\tcomment "') and line != '\tcomment "System"': + sections.append((line[len('\tcomment "'):].rstrip('"'), [])) + elif line.startswith("\trom (") and sections: + sections[-1][1].append(line) + return sections + + class Exporter(BaseExporter): """Write libretro's System.dat, corrected.""" @@ -118,16 +135,17 @@ class Exporter(BaseExporter): ) -> dict[str, str]: lines = self._header(originals, scraped) lines.extend(["game (", '\tname "System"', '\tcomment "System"']) - quoted = frozenset(_QUOTED_NAME.findall(originals.get(self.native_filename(), ""))) + original = originals.get(self.native_filename(), "") + quoted = frozenset(_QUOTED_NAME.findall(original)) - for system, files in sorted( - self.exportable(systems), key=lambda pair: pair[0].native_id - ): - rendered: list[str] = [] + ours: dict[str, dict[str, str]] = {} + for system, files in self.exportable(systems): + rendered: dict[str, str] = {} for fe in files: if not any(fe.hash(h) for h in ("crc32", "md5", "sha1")): continue - parts = [f"name {_quote(self._rom_name(fe), quoted)}"] + name = _quote(self._rom_name(fe), quoted) + parts = [f"name {name}"] size = fe.size() if size: parts.append(f"size {size}") @@ -140,15 +158,29 @@ class Exporter(BaseExporter): sha1 = fe.hash("sha1") if sha1: parts.append(f"sha1 {sha1}") - rendered.append(f"\trom ( {' '.join(parts)} )") + rendered.setdefault(name, f"\trom ( {' '.join(parts)} )") + if rendered: + ours[system.native_id] = rendered - if not rendered: - continue + def section(comment: str, roms: list[str]) -> None: lines.append("") # libretro's comment is the system name as the DAT spells it, # "Atari - 400-800". Prettifying it drops the separator. - lines.append(f'\tcomment "{system.native_id}"') - lines.extend(rendered) + lines.append(f'\tcomment "{comment}"') + lines.extend(roms) + + # libretro's own sections are the base: a rom line our transcription + # does not carry (added or renamed after a scrape) stays as written. + for comment, roms in _sections(original): + rendered = ours.pop(comment, {}) + body = [] + for line in roms: + match = _ROM_NAME.match(line) + body.append(rendered.pop(match.group(1), line) if match else line) + body.extend(rendered.values()) + section(comment, body) + for comment in sorted(ours): + section(comment, list(ours[comment].values())) lines.append(")") lines.append("") diff --git a/tests/test_export_counts.py b/tests/test_export_counts.py index 6ed3a51a..3e6eed07 100644 --- a/tests/test_export_counts.py +++ b/tests/test_export_counts.py @@ -28,6 +28,7 @@ from exporter.retrobat_exporter import Exporter as RetroBat # noqa: E402 from exporter.retrodeck_exporter import Exporter as RetroDeck # noqa: E402 from exporter.recalbox_exporter import Exporter as Recalbox # noqa: E402 from exporter.retropie_exporter import Exporter as RetroPie # noqa: E402 +from exporter.systemdat_exporter import Exporter as SystemDat # noqa: E402 A = "a" * 32 B = "b" * 32 @@ -181,6 +182,66 @@ class RetroBatKeepsWhatWeDoNotModel(unittest.TestCase): ]) +class PureDataExportsKeepWhatWeDoNotModel(unittest.TestCase): + """Recalbox's es_bios.xml and libretro's System.dat were rebuilt from our + model too: an entry the platform added or renamed after our scrape left + the platform's own file.""" + + @staticmethod + def _file(name, system, **platform): + return NativeFile(name, name, system, platform={"md5": A, **platform}, + truth={"md5": A, "crc32": "1234abcd", "sha1": "f" * 40, + "size": 4}, + filled=[]) + + def test_recalbox(self): + from exporter.baseline import Report + from exporter.recalbox_exporter import Exporter as Recalbox + + fe = self._file("a.bin", "nes", destination="nes/a.bin", core="libretro:fceumm") + systems = {"nes": NativeSystem("nes", files=[fe])} + original = ( + '\n\n' + ' \n' + f' \n' + f' \n' + ' \n' + ' \n' + f' \n' + ' \n\n' + ) + produced = Recalbox().render(systems, Report(), {"es_bios.xml": original})["es_bios.xml"] + from xml.etree import ElementTree + + root = ElementTree.fromstring(produced) + found = [(s.get("platform"), b.get("path"), b.get("md5")) + for s in root.iter("system") for b in s.iter("bios")] + self.assertEqual(found, [("nes", "nes/new.bin", B), ("nes", "nes/a.bin", A), + ("newsys", "n.bin", B)]) + self.assertIn('mandatory="false"', produced) + + def test_system_dat(self): + from exporter.baseline import Report + + fe = self._file("a.bin", "Nintendo - NES") + systems = {"Nintendo - NES": NativeSystem("Nintendo - NES", files=[fe])} + original = "\n".join([ + "clrmamepro (", '\tname "System"', ")", "", "game (", '\tname "System"', + '\tcomment "System"', "", + '\tcomment "Nintendo - NES"', + "\trom ( name new.bin size 2 crc 00000000 )", + "\trom ( name a.bin size 4 crc DEADBEEF )", "", + '\tcomment "New - System"', + "\trom ( name n.bin size 1 crc 11111111 )", ")", "", + ]) + produced = SystemDat().render(systems, Report(), {"System.dat": original})["System.dat"] + roms = [line for line in produced.splitlines() if line.startswith("\trom")] + self.assertEqual(roms[0], "\trom ( name new.bin size 2 crc 00000000 )") + self.assertIn("name a.bin size 4 crc 1234ABCD", roms[1]) + self.assertEqual(roms[2], "\trom ( name n.bin size 1 crc 11111111 )") + self.assertIn('\tcomment "New - System"', produced) + + class EmuDeckWritesItsCorrections(unittest.TestCase): """A correction replaced nothing: every withdrawal was refused, the run counted the correction anyway and validate failed on it."""