"""Device ROM sets read from a MAME -listxml and merged into a profile. Synthetic listxml and driver source, no network: the scan keeps only device sets with dumped ROMs, the source line is the ROM_START block holding them, and the merge adds sets, refreshes the contents of written ones without touching what a reader wrote, and leaves BIOS root sets alone. """ from __future__ import annotations import io import json import tempfile import unittest from pathlib import Path from scripts.common import yaml_load from scripts.scraper._hash_merge import ( DEVICE_SECTION, mame_ref_role, merge_mame_devices, merge_mame_profile, ) from scripts.scraper.mame_devices import ( ABSENT, AMBIGUOUS, NO_FILE, find_rom_block, locate_sources, open_listxml, scan_device_sets, source_path, ) LISTXML = b""" ]> \t \t\tNeo-Geo MV-6F \t\t \t \t \t\tQSound \t\t \t \t \t\tKeyboard & Mouse - New \t\t \t\t \t\t \t \t \t\tUndumped \t\t \t \t \t\tNo ROM \t """ KBD_SOURCE = """// keyboards ROM_START(kbd_old) \tROM_REGION(0x0800, "mcu", 0) \tROM_LOAD("6500-1", 0x0000, 0x0800, CRC(4a3fc332) SHA1(83b21d0c8b93fc9b9b3b287fde4ec8f3badac5a2) BAD_DUMP) \tROM_LOAD("bad.bin", 0x0800, 0x0010, CRC(00000011) SHA1(0000000000000000000000000000000000000011)) ROM_END ROM_START(kbd_new) \tROM_REGION(0x0800, "mcu", 0) \tROM_LOAD("6570-036", 0x0000, 0x0800, CRC(4a3fc332) SHA1(83b21d0c8b93fc9b9b3b287fde4ec8f3badac5a2)) \tROM_LOAD("bad.bin", 0x0800, 0x0010, CRC(00000011) SHA1(0000000000000000000000000000000000000011)) ROM_END """.splitlines() QSOUND_SOURCE = """#define QSOUND_ROMS \\ \tROM_LOAD16_WORD_SWAP("dl-1425.bin", 0x0000, 0x6000, CRC(d6cf5ef5) SHA1(555f50fe5cdf127619da7d854c03f4a244a0c501)) // ROM_START( commented ) CRC(d6cf5ef5) ROM_START( qsound ) \tROM_REGION16_BE( 0x2000, "dsp", 0 ) \tQSOUND_ROMS ROM_END """.splitlines() PROFILE = """emulator: MAME source: "https://github.com/libretro/mame" upstream: "https://github.com/mamedev/mame" upstream_commit: "f34f02505e32c1993c6a782b6814232cbfc74e36" core_version: "0.289" files: # SNK Neo Geo - name: neogeo.zip required: true category: bios_zip system: snk-neogeo-mvs source_ref: "upstream src/mame/snk/neogeo.cpp:2400" contents: - name: "sp-s2.sp1" description: "Europe MVS (Ver. 2)" size: 131072 crc32: "oldcrc32" - name: qsound.zip required: true category: bios_zip system: capcom-cps2 source_ref: "upstream src/devices/sound/qsound.cpp:121-125" note: >- Written by hand. contents: - name: "dl-1425.bin" description: "QSound DSP microcode" size: 24576 crc32: "d6cf5ef5" """ def _devices() -> dict[str, dict]: tag, devices = scan_device_sets(io.BytesIO(LISTXML)) assert tag == "mame0289" return devices def _fetch(path: str) -> list[str] | None: return { "src/devices/kbd.cpp": KBD_SOURCE, "src/devices/sound/qsound.cpp": QSOUND_SOURCE, }.get(path) class ScanKeepsDeviceSetsWithDumpedRoms(unittest.TestCase): def test_only_devices_with_a_crc_are_kept(self): devices = _devices() self.assertEqual(sorted(devices), ["kbd_new", "qsound"]) def test_rom_fields_bios_description_and_bad_dump(self): kbd = _devices()["kbd_new"] self.assertEqual(kbd["description"], "Keyboard & Mouse - New") self.assertEqual(kbd["sourcefile"], "devices/kbd.cpp") first, second = kbd["roms"] self.assertEqual( (first["name"], first["size"], first["crc32"], first["bios_description"]), ("6570-036", 2048, "4a3fc332", "Version 2"), ) self.assertTrue(second["bad_dump"]) def test_a_document_without_a_build_is_refused(self): with self.assertRaises(ValueError): scan_device_sets(io.BytesIO(b"")) def test_the_release_zip_is_read(self): import zipfile with tempfile.TemporaryDirectory() as tmp: pack = Path(tmp) / "mame0289lx.zip" with zipfile.ZipFile(pack, "w") as archive: archive.writestr("mame0289.xml", LISTXML) with open_listxml(pack) as stream: tag, devices = scan_device_sets(stream) self.assertEqual((tag, sorted(devices)), ("mame0289", ["kbd_new", "qsound"])) def test_source_paths(self): self.assertEqual(source_path("devices/sound/qsound.cpp"), "src/devices/sound/qsound.cpp") self.assertEqual(source_path("snk/neogeo.cpp"), "src/mame/snk/neogeo.cpp") class SourceLinesAreTheBlockHoldingTheRoms(unittest.TestCase): def test_found_through_a_rom_macro_past_a_comment(self): self.assertEqual( find_rom_block(QSOUND_SOURCE, "qsound", [("dl-1425.bin", "d6cf5ef5")]), ("5-8", ""), ) def test_a_block_inside_a_comment_is_not_a_block(self): source = [ "/* ROM_START( old )", '\tROM_LOAD("dl-1425.bin", 0, 0x6000, CRC(d6cf5ef5) SHA1(555f50fe5cdf127619da7d854c03f4a244a0c501))', "ROM_END */", "ROM_START( current )", '\tROM_LOAD("dl-1425.bin", 0, 0x6000, CRC(d6cf5ef5) SHA1(555f50fe5cdf127619da7d854c03f4a244a0c501))', "ROM_END", ] self.assertEqual( find_rom_block(source, "dsp", [("dl-1425.bin", "d6cf5ef5")]), ("4-6", "") ) def test_same_dump_under_two_names_is_told_apart_by_name(self): line, _ = find_rom_block( KBD_SOURCE, "kbd_new", [("6570-036", "4a3fc332"), ("bad.bin", "00000011")] ) self.assertEqual(line, "8-12") def test_not_found(self): self.assertEqual( find_rom_block(KBD_SOURCE, "x", [("x.bin", "deadbeef")]), (None, ABSENT) ) def test_two_blocks_and_no_way_to_choose(self): self.assertEqual( find_rom_block(KBD_SOURCE, "other", [("bad.bin", "00000011")]), (None, AMBIGUOUS), ) def test_the_label_device_rom_region_returns_settles_it(self): source = KBD_SOURCE + [ 'DEFINE_DEVICE_TYPE(KBD_X, kbd_x_device, "kbd_x", "Keyboard X")', "const tiny_rom_entry *kbd_x_device::device_rom_region() const", "{", "\treturn ROM_NAME(kbd_old);", "}", ] self.assertEqual( find_rom_block(source, "kbd_x", [("bad.bin", "00000011")]), ("2-6", "") ) def test_locate_sources_reports_what_it_cannot_place(self): devices = _devices() devices["ghost"] = { "description": "Ghost", "sourcefile": "devices/ghost.cpp", "roms": [{"name": "g.bin", "size": 1, "crc32": "00000001", "sha1": ""}], } unplaced = locate_sources(devices, _fetch, workers=2) self.assertEqual(unplaced, {"ghost": NO_FILE}) self.assertEqual(devices["qsound"]["source_file"], "src/devices/sound/qsound.cpp") self.assertEqual(devices["kbd_new"]["source_line"], "8-12") class MergeIntoTheProfile(unittest.TestCase): def setUp(self): self._tmp = tempfile.TemporaryDirectory() self.path = Path(self._tmp.name) / "mame.yml" self.path.write_text(PROFILE, encoding="utf-8") self.devices = _devices() locate_sources(self.devices, _fetch, workers=2) def tearDown(self): self._tmp.cleanup() def _merge(self) -> dict[str, list[str]]: return merge_mame_devices(str(self.path), self.devices, write=True) def _entries(self) -> dict[str, dict]: data = yaml_load(self.path.read_text(encoding="utf-8")) return {f["name"]: f for f in data["files"]} def test_a_device_with_roms_is_added_under_the_section(self): report = self._merge() self.assertEqual(report["added"], ["kbd_new"]) text = self.path.read_text(encoding="utf-8") self.assertLess(text.index(DEVICE_SECTION), text.index("- name: kbd_new.zip")) entry = self._entries()["kbd_new.zip"] self.assertIs(entry["required"], False) self.assertEqual(entry["category"], "bios_zip") self.assertEqual(entry["source_ref"], "upstream src/devices/kbd.cpp:8-12") self.assertEqual( entry["note"], "Device ROM set, loaded when a machine instantiates `Keyboard & Mouse - New`.", ) self.assertEqual(entry["contents"][0]["description"], "`Version 2`") self.assertEqual(entry["contents"][0]["sha1"], "83b21d0c8b93fc9b9b3b287fde4ec8f3badac5a2") self.assertIs(entry["contents"][1]["bad_dump"], True) def test_devices_without_roms_are_not_written(self): self._merge() entries = self._entries() self.assertNotIn("nodump_dev.zip", entries) self.assertNotIn("empty_dev.zip", entries) def test_an_unplaced_set_is_written_without_a_line(self): del self.devices["kbd_new"]["source_line"] self._merge() self.assertNotIn("source_ref", self._entries()["kbd_new.zip"]) def test_a_written_entry_gets_its_contents_and_keeps_the_rest(self): report = self._merge() self.assertEqual(report["updated"], ["qsound"]) entry = self._entries()["qsound.zip"] self.assertIs(entry["required"], True) self.assertEqual(entry["system"], "capcom-cps2") self.assertEqual(entry["note"], "Written by hand.") self.assertEqual(entry["source_ref"], "upstream src/devices/sound/qsound.cpp:121-125") member = entry["contents"][0] self.assertEqual(member["description"], "QSound DSP microcode") self.assertEqual(member["sha1"], "555f50fe5cdf127619da7d854c03f4a244a0c501") def test_a_written_entry_without_a_ref_gets_the_found_one(self): self.path.write_text( PROFILE.replace(' source_ref: "upstream src/devices/sound/qsound.cpp:121-125"\n', ""), encoding="utf-8", ) self._merge() self.assertEqual( self._entries()["qsound.zip"]["source_ref"], "upstream src/devices/sound/qsound.cpp:5-8", ) def test_bios_roots_are_left_as_written(self): before = PROFILE[PROFILE.index(" # SNK"):PROFILE.index(" - name: qsound.zip")] self._merge() self.assertIn(before, self.path.read_text(encoding="utf-8")) def test_a_second_run_changes_nothing(self): self._merge() first = self.path.read_text(encoding="utf-8") report = self._merge() self.assertEqual((report["added"], report["updated"]), ([], [])) self.assertEqual(self.path.read_text(encoding="utf-8"), first) def test_new_sets_keep_name_order_inside_the_section(self): self._merge() self.devices["aaa_dev"] = dict(self.devices["kbd_new"]) self.devices["zzz_dev"] = dict(self.devices["kbd_new"]) self._merge() text = self.path.read_text(encoding="utf-8") order = [text.index(f"- name: {n}.zip") for n in ("aaa_dev", "kbd_new", "zzz_dev")] self.assertEqual(order, sorted(order)) self.assertTrue(text.endswith(' bad_dump: true\n')) def test_another_category_under_a_device_name_is_reported(self): self.path.write_text( PROFILE + "\n - name: kbd_new.zip\n category: game_data\n", encoding="utf-8" ) report = self._merge() self.assertEqual(report["conflict"], ["kbd_new"]) def test_a_new_bios_root_lands_before_the_device_section(self): self._merge() hashes = Path(self._tmp.name) / "h.json" hashes.write_text(json.dumps({ "version": "0.289", "commit": "f34f02505e32c1993c6a782b6814232cbfc74e36", "bios_sets": {"newbios": { "source_file": "src/mame/x/x.cpp", "source_line": 7, "roms": [{"name": "n.bin", "size": 4, "crc32": "12345678"}], }}, }), encoding="utf-8") merge_mame_profile(str(self.path), str(hashes), write=True) text = self.path.read_text(encoding="utf-8") self.assertLess(text.index("newbios.zip"), text.index(DEVICE_SECTION)) self.assertEqual(self._entries()["newbios.zip"]["source_ref"], "upstream src/mame/x/x.cpp:7") self.assertIn("kbd_new.zip", self._entries()) class RefRole(unittest.TestCase): def test_a_port_cites_mamedev_as_upstream(self): self.assertEqual(mame_ref_role({ "source": "https://github.com/libretro/mame", "upstream": "https://github.com/mamedev/mame", }), "upstream ") def test_a_profile_with_one_tree_cites_it_bare(self): self.assertEqual(mame_ref_role({"emulator": "MAME"}), "") class Cli(unittest.TestCase): def test_devices_needs_a_listxml(self): from scripts.scraper.mame_hash_scraper import build_parser, main # noqa: F401 import sys from unittest import mock with mock.patch.object(sys, "argv", ["x", "--devices"]), \ mock.patch("sys.stderr", new=io.StringIO()): with self.assertRaises(SystemExit) as raised: main() self.assertEqual(raised.exception.code, 2) if __name__ == "__main__": unittest.main()