diff --git a/scripts/exporter/recalbox_exporter.py b/scripts/exporter/recalbox_exporter.py
index b85ccf9f..9e5aa1fe 100644
--- a/scripts/exporter/recalbox_exporter.py
+++ b/scripts/exporter/recalbox_exporter.py
@@ -115,18 +115,56 @@ class Exporter(BaseExporter):
'',
]
+ # Recalbox's own file is the base, so an element our transcription
+ # does not carry (added or renamed after a scrape) stays as written,
+ # and its system order with it.
+ theirs: list[tuple[str, str, list[dict[str, str]]]] = []
+ original = originals.get(self.native_filename(), "")
+ if original:
+ try:
+ root = parse_untrusted_xml(original, self.native_filename())
+ except (ParseError, ValueError):
+ root = None
+ if root is not None:
+ theirs = [
+ (s.get("platform", ""), s.get("fullname", ""),
+ [dict(b.attrib) for b in s.iter("bios")])
+ for s in root.iter("system")
+ ]
- for system in sorted(systems.values(), key=lambda s: s.native_id):
+ ours: dict[str, tuple[NativeSystem, dict[str, str]]] = {}
+ for system in systems.values():
writable = [fe for fe in system.files if self.writable(fe)]
- if not writable:
- continue
- fullname = quoteattr(self.display_name(system))
- platform = quoteattr(system.native_id)
- lines.append(f" ")
- for fe in writable:
- lines.append(self._bios_element(fe, system.native_id))
+ if writable:
+ ours[system.native_id] = (system, {
+ self._path(fe): self._bios_element(fe, system.native_id)
+ for fe in writable
+ })
+
+ def element(attrs: dict[str, str]) -> str:
+ rendered = " ".join(f"{k}={quoteattr(v)}" for k, v in attrs.items())
+ return f" "
+
+ def emit(platform: str, fullname: str, body: list[str]) -> None:
+ lines.append(
+ f" "
+ )
+ lines.extend(body)
lines.append(" ")
+ for platform, fullname, bios in theirs:
+ if platform not in ours:
+ emit(platform, fullname, [element(attrs) for attrs in bios])
+ continue
+ system, elements = ours.pop(platform)
+ body = [elements.pop(attrs.get("path", ""), None) or element(attrs)
+ for attrs in bios]
+ body.extend(elements.values())
+ emit(platform, self.display_name(system), body)
+ for platform in sorted(ours):
+ system, elements = ours[platform]
+ emit(platform, self.display_name(system), list(elements.values()))
+
lines.append("")
lines.append("")
return {self.native_filename(): "\n".join(lines)}
diff --git a/scripts/exporter/systemdat_exporter.py b/scripts/exporter/systemdat_exporter.py
index f60db66b..3b198490 100644
--- a/scripts/exporter/systemdat_exporter.py
+++ b/scripts/exporter/systemdat_exporter.py
@@ -36,6 +36,23 @@ def _quote(name: str, quoted: frozenset[str] = frozenset()) -> str:
return f'"{name}"' if name in quoted or any(c in name for c in ' ()') else name
+_ROM_NAME = re.compile(r'^\trom \( name ("[^"]+"|\S+)')
+
+
+def _sections(original: str) -> list[tuple[str, list[str]]]:
+ """The system sections of a System.dat, each comment with its rom lines."""
+ _, sep, body = original.partition("\ngame (")
+ if not sep:
+ return []
+ sections: list[tuple[str, list[str]]] = []
+ for line in body.splitlines():
+ if line.startswith('\tcomment "') and line != '\tcomment "System"':
+ sections.append((line[len('\tcomment "'):].rstrip('"'), []))
+ elif line.startswith("\trom (") and sections:
+ sections[-1][1].append(line)
+ return sections
+
+
class Exporter(BaseExporter):
"""Write libretro's System.dat, corrected."""
@@ -118,16 +135,17 @@ class Exporter(BaseExporter):
) -> dict[str, str]:
lines = self._header(originals, scraped)
lines.extend(["game (", '\tname "System"', '\tcomment "System"'])
- quoted = frozenset(_QUOTED_NAME.findall(originals.get(self.native_filename(), "")))
+ original = originals.get(self.native_filename(), "")
+ quoted = frozenset(_QUOTED_NAME.findall(original))
- for system, files in sorted(
- self.exportable(systems), key=lambda pair: pair[0].native_id
- ):
- rendered: list[str] = []
+ ours: dict[str, dict[str, str]] = {}
+ for system, files in self.exportable(systems):
+ rendered: dict[str, str] = {}
for fe in files:
if not any(fe.hash(h) for h in ("crc32", "md5", "sha1")):
continue
- parts = [f"name {_quote(self._rom_name(fe), quoted)}"]
+ name = _quote(self._rom_name(fe), quoted)
+ parts = [f"name {name}"]
size = fe.size()
if size:
parts.append(f"size {size}")
@@ -140,15 +158,29 @@ class Exporter(BaseExporter):
sha1 = fe.hash("sha1")
if sha1:
parts.append(f"sha1 {sha1}")
- rendered.append(f"\trom ( {' '.join(parts)} )")
+ rendered.setdefault(name, f"\trom ( {' '.join(parts)} )")
+ if rendered:
+ ours[system.native_id] = rendered
- if not rendered:
- continue
+ def section(comment: str, roms: list[str]) -> None:
lines.append("")
# libretro's comment is the system name as the DAT spells it,
# "Atari - 400-800". Prettifying it drops the separator.
- lines.append(f'\tcomment "{system.native_id}"')
- lines.extend(rendered)
+ lines.append(f'\tcomment "{comment}"')
+ lines.extend(roms)
+
+ # libretro's own sections are the base: a rom line our transcription
+ # does not carry (added or renamed after a scrape) stays as written.
+ for comment, roms in _sections(original):
+ rendered = ours.pop(comment, {})
+ body = []
+ for line in roms:
+ match = _ROM_NAME.match(line)
+ body.append(rendered.pop(match.group(1), line) if match else line)
+ body.extend(rendered.values())
+ section(comment, body)
+ for comment in sorted(ours):
+ section(comment, list(ours[comment].values()))
lines.append(")")
lines.append("")
diff --git a/tests/test_export_counts.py b/tests/test_export_counts.py
index 6ed3a51a..3e6eed07 100644
--- a/tests/test_export_counts.py
+++ b/tests/test_export_counts.py
@@ -28,6 +28,7 @@ from exporter.retrobat_exporter import Exporter as RetroBat # noqa: E402
from exporter.retrodeck_exporter import Exporter as RetroDeck # noqa: E402
from exporter.recalbox_exporter import Exporter as Recalbox # noqa: E402
from exporter.retropie_exporter import Exporter as RetroPie # noqa: E402
+from exporter.systemdat_exporter import Exporter as SystemDat # noqa: E402
A = "a" * 32
B = "b" * 32
@@ -181,6 +182,66 @@ class RetroBatKeepsWhatWeDoNotModel(unittest.TestCase):
])
+class PureDataExportsKeepWhatWeDoNotModel(unittest.TestCase):
+ """Recalbox's es_bios.xml and libretro's System.dat were rebuilt from our
+ model too: an entry the platform added or renamed after our scrape left
+ the platform's own file."""
+
+ @staticmethod
+ def _file(name, system, **platform):
+ return NativeFile(name, name, system, platform={"md5": A, **platform},
+ truth={"md5": A, "crc32": "1234abcd", "sha1": "f" * 40,
+ "size": 4},
+ filled=[])
+
+ def test_recalbox(self):
+ from exporter.baseline import Report
+ from exporter.recalbox_exporter import Exporter as Recalbox
+
+ fe = self._file("a.bin", "nes", destination="nes/a.bin", core="libretro:fceumm")
+ systems = {"nes": NativeSystem("nes", files=[fe])}
+ original = (
+ '\n\n'
+ ' \n'
+ f' \n'
+ f' \n'
+ ' \n'
+ ' \n'
+ f' \n'
+ ' \n\n'
+ )
+ produced = Recalbox().render(systems, Report(), {"es_bios.xml": original})["es_bios.xml"]
+ from xml.etree import ElementTree
+
+ root = ElementTree.fromstring(produced)
+ found = [(s.get("platform"), b.get("path"), b.get("md5"))
+ for s in root.iter("system") for b in s.iter("bios")]
+ self.assertEqual(found, [("nes", "nes/new.bin", B), ("nes", "nes/a.bin", A),
+ ("newsys", "n.bin", B)])
+ self.assertIn('mandatory="false"', produced)
+
+ def test_system_dat(self):
+ from exporter.baseline import Report
+
+ fe = self._file("a.bin", "Nintendo - NES")
+ systems = {"Nintendo - NES": NativeSystem("Nintendo - NES", files=[fe])}
+ original = "\n".join([
+ "clrmamepro (", '\tname "System"', ")", "", "game (", '\tname "System"',
+ '\tcomment "System"', "",
+ '\tcomment "Nintendo - NES"',
+ "\trom ( name new.bin size 2 crc 00000000 )",
+ "\trom ( name a.bin size 4 crc DEADBEEF )", "",
+ '\tcomment "New - System"',
+ "\trom ( name n.bin size 1 crc 11111111 )", ")", "",
+ ])
+ produced = SystemDat().render(systems, Report(), {"System.dat": original})["System.dat"]
+ roms = [line for line in produced.splitlines() if line.startswith("\trom")]
+ self.assertEqual(roms[0], "\trom ( name new.bin size 2 crc 00000000 )")
+ self.assertIn("name a.bin size 4 crc 1234ABCD", roms[1])
+ self.assertEqual(roms[2], "\trom ( name n.bin size 1 crc 11111111 )")
+ self.assertIn('\tcomment "New - System"', produced)
+
+
class EmuDeckWritesItsCorrections(unittest.TestCase):
"""A correction replaced nothing: every withdrawal was refused, the run
counted the correction anyway and validate failed on it."""