"""Tests for the hash merge module.""" from __future__ import annotations import json import tempfile import unittest from pathlib import Path import yaml from scripts.scraper._hash_merge import ( _build_source_ref, compute_diff, merge_fbneo_profile, merge_mame_profile, ) def _write_yaml(path: Path, data: dict) -> str: p = str(path) with open(p, "w", encoding="utf-8") as f: yaml.dump(data, f, default_flow_style=False, sort_keys=False) return p def _write_json(path: Path, data: dict) -> str: p = str(path) with open(p, "w", encoding="utf-8") as f: json.dump(data, f) return p def _make_mame_profile(**overrides: object) -> dict: base = { "emulator": "MAME", "core_version": "0.285", "files": [ { "name": "neogeo.zip", "required": True, "category": "bios_zip", "system": "snk-neogeo-mvs", "source_ref": "src/mame/neogeo/neogeo.cpp:2400", "contents": [ { "name": "sp-s2.sp1", "size": 131072, "crc32": "oldcrc32", "description": "Europe MVS (Ver. 2)", }, ], }, ], } base.update(overrides) return base def _make_mame_hashes(**overrides: object) -> dict: base = { "source": "mamedev/mame", "version": "0.286", "commit": "abc123", "fetched_at": "2026-03-30T12:00:00Z", "bios_sets": { "neogeo": { "source_file": "src/mame/neogeo/neogeo.cpp", "source_line": 2432, "roms": [ { "name": "sp-s2.sp1", "size": 131072, "crc32": "9036d879", "sha1": "4f834c55", "region": "mainbios", "bios_label": "euro", "bios_description": "Europe MVS (Ver. 2)", }, ], }, }, } base.update(overrides) return base def _make_fbneo_profile(**overrides: object) -> dict: base = { "emulator": "FinalBurn Neo", "core_version": "v1.0.0.02", "files": [ { "name": "sp-s2.sp1", "archive": "neogeo.zip", "system": "snk-neogeo-mvs", "required": True, "size": 131072, "crc32": "oldcrc32", "source_ref": "src/burn/drv/neogeo/d_neogeo.cpp:1605", }, { "name": "hiscore.dat", "required": False, }, ], } base.update(overrides) return base def _make_fbneo_hashes(**overrides: object) -> dict: base = { "source": "finalburnneo/FBNeo", "version": "v1.0.0.03", "commit": "def456", "fetched_at": "2026-03-30T12:00:00Z", "bios_sets": { "neogeo": { "source_file": "src/burn/drv/neogeo/d_neogeo.cpp", "source_line": 1604, "roms": [ { "name": "sp-s2.sp1", "size": 131072, "crc32": "9036d879", "sha1": "aabbccdd", }, ], }, }, } base.update(overrides) return base class TestMameMerge(unittest.TestCase): """Tests for merge_mame_profile.""" def test_merge_updates_contents(self) -> None: with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes()) result = merge_mame_profile(profile_path, hashes_path) bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"] self.assertEqual(len(bios_files), 1) contents = bios_files[0]["contents"] self.assertEqual(contents[0]["crc32"], "9036d879") self.assertEqual(contents[0]["sha1"], "4f834c55") self.assertEqual(contents[0]["description"], "Europe MVS (Ver. 2)") def test_merge_preserves_manual_fields(self) -> None: profile = _make_mame_profile() profile["files"][0]["note"] = "manually curated note" profile["files"][0]["system"] = "snk-neogeo-mvs" profile["files"][0]["required"] = False with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", profile) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes()) result = merge_mame_profile(profile_path, hashes_path) entry = [f for f in result["files"] if f.get("category") == "bios_zip"][0] self.assertEqual(entry["note"], "manually curated note") self.assertEqual(entry["system"], "snk-neogeo-mvs") self.assertFalse(entry["required"]) def test_merge_adds_new_bios_set(self) -> None: hashes = _make_mame_hashes() hashes["bios_sets"]["pgm"] = { "source_file": "src/mame/igs/pgm.cpp", "source_line": 5515, "roms": [ {"name": "pgm_t01s.rom", "size": 2097152, "crc32": "1a7123a0"}, ], } with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile()) hashes_path = _write_json(p / "hashes.json", hashes) result = merge_mame_profile(profile_path, hashes_path) bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"] names = {f["name"] for f in bios_files} self.assertIn("pgm.zip", names) pgm = next(f for f in bios_files if f["name"] == "pgm.zip") self.assertIsNone(pgm["system"]) self.assertTrue(pgm["required"]) self.assertEqual(pgm["category"], "bios_zip") def test_numeric_looking_rom_name_survives_the_write(self) -> None: """MAME ships ROMs named 01 and 23. Bare YAML turns them into numbers. The manager set loads "01"; written unquoted it re-parses as the integer 1 and the profile stops naming the file the driver opens. """ from scripts.scraper._hash_merge import ( _append_new_entries, _format_contents, ) contents = [ {"name": "01", "description": "ROM bank 0-1", "size": 8192, "crc32": "702f4cf5"}, {"name": "23", "description": "ROM bank 2-3", "size": 8192, "crc32": "46489d88"}, ] document = yaml.safe_load( "files:\n - name: manager.zip\n" + _format_contents(contents) ) names = [rom["name"] for rom in document["files"][0]["contents"]] self.assertEqual(names, ["01", "23"]) for name in names: self.assertIsInstance(name, str) original = "emulator: MAME\nfiles:\n - name: neogeo.zip\n" appended = _append_new_entries( original, [{ "name": "manager.zip", "required": False, "category": "bios_zip", "source_ref": "src/mame/vtech/crvision.cpp:955-959", "contents": contents, }], original, ) entry = next( f for f in yaml.safe_load(appended)["files"] if f["name"] == "manager.zip" ) self.assertEqual( [rom["name"] for rom in entry["contents"]], ["01", "23"] ) def test_merge_preserves_non_bios_files(self) -> None: profile = _make_mame_profile() profile["files"].append({"name": "hiscore.dat", "required": False}) with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", profile) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes()) result = merge_mame_profile(profile_path, hashes_path) non_bios = [f for f in result["files"] if f.get("category") != "bios_zip"] self.assertEqual(len(non_bios), 1) self.assertEqual(non_bios[0]["name"], "hiscore.dat") def test_merge_keeps_unmatched_bios_set(self) -> None: """Entries not in scraper scope stay untouched (no _upstream_removed).""" hashes = _make_mame_hashes() hashes["bios_sets"] = {} # nothing from scraper with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile()) hashes_path = _write_json(p / "hashes.json", hashes) result = merge_mame_profile(profile_path, hashes_path) bios_files = [f for f in result["files"] if f.get("category") == "bios_zip"] self.assertEqual(len(bios_files), 1) self.assertNotIn("_upstream_removed", bios_files[0]) self.assertEqual(bios_files[0]["name"], "neogeo.zip") def test_merge_updates_core_version(self) -> None: with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes()) result = merge_mame_profile(profile_path, hashes_path) self.assertEqual(result["core_version"], "0.286") def test_merge_backup_created(self) -> None: with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes()) merge_mame_profile(profile_path, hashes_path, write=True) backup = p / "mame.old.yml" self.assertTrue(backup.exists()) with open(backup, encoding="utf-8") as f: old = yaml.safe_load(f) self.assertEqual(old["core_version"], "0.285") def test_merge_updates_source_ref(self) -> None: with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile()) hashes_path = _write_json(p / "hashes.json", _make_mame_hashes()) result = merge_mame_profile(profile_path, hashes_path) entry = [f for f in result["files"] if f.get("category") == "bios_zip"][0] self.assertEqual(entry["source_ref"], "src/mame/neogeo/neogeo.cpp:2432") class TestFbneoMerge(unittest.TestCase): """Tests for merge_fbneo_profile.""" def test_merge_updates_rom_entries(self) -> None: with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile()) hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes()) result = merge_fbneo_profile(profile_path, hashes_path) archive_files = [f for f in result["files"] if "archive" in f] self.assertEqual(len(archive_files), 1) self.assertEqual(archive_files[0]["crc32"], "9036d879") self.assertEqual(archive_files[0]["system"], "snk-neogeo-mvs") def test_merge_adds_new_roms(self) -> None: hashes = _make_fbneo_hashes() hashes["bios_sets"]["neogeo"]["roms"].append( { "name": "sp-s3.sp1", "size": 131072, "crc32": "91b64be3", } ) with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile()) hashes_path = _write_json(p / "hashes.json", hashes) result = merge_fbneo_profile(profile_path, hashes_path) archive_files = [f for f in result["files"] if "archive" in f] self.assertEqual(len(archive_files), 2) new_rom = next(f for f in archive_files if f["name"] == "sp-s3.sp1") self.assertEqual(new_rom["archive"], "neogeo.zip") self.assertTrue(new_rom["required"]) def test_merge_preserves_non_archive_files(self) -> None: with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile()) hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes()) result = merge_fbneo_profile(profile_path, hashes_path) non_archive = [f for f in result["files"] if "archive" not in f] self.assertEqual(len(non_archive), 1) self.assertEqual(non_archive[0]["name"], "hiscore.dat") def test_merge_keeps_unmatched_roms(self) -> None: """Entries not in scraper scope stay untouched (no _upstream_removed).""" hashes = _make_fbneo_hashes() hashes["bios_sets"] = {} with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile()) hashes_path = _write_json(p / "hashes.json", hashes) result = merge_fbneo_profile(profile_path, hashes_path) archive_files = [f for f in result["files"] if "archive" in f] self.assertEqual(len(archive_files), 1) self.assertNotIn("_upstream_removed", archive_files[0]) def test_merge_updates_core_version(self) -> None: with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile()) hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes()) result = merge_fbneo_profile(profile_path, hashes_path) self.assertEqual(result["core_version"], "v1.0.0.03") class TestDiff(unittest.TestCase): """Tests for compute_diff.""" def test_diff_mame_detects_changes(self) -> None: hashes = _make_mame_hashes() hashes["bios_sets"]["pgm"] = { "source_file": "src/mame/igs/pgm.cpp", "source_line": 5515, "roms": [ {"name": "pgm_t01s.rom", "size": 2097152, "crc32": "1a7123a0"}, ], } with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile()) hashes_path = _write_json(p / "hashes.json", hashes) diff = compute_diff(profile_path, hashes_path, mode="mame") self.assertIn("pgm", diff["added"]) self.assertIn("neogeo", diff["updated"]) self.assertEqual(len(diff["removed"]), 0) self.assertEqual(diff["unchanged"], 0) def test_diff_mame_out_of_scope(self) -> None: """Items in profile but not in scraper output = out of scope, not removed.""" hashes = _make_mame_hashes() hashes["bios_sets"] = {} with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "mame.yml", _make_mame_profile()) hashes_path = _write_json(p / "hashes.json", hashes) diff = compute_diff(profile_path, hashes_path, mode="mame") self.assertEqual(diff["removed"], []) self.assertEqual(diff["out_of_scope"], 1) self.assertEqual(len(diff["added"]), 0) def test_diff_fbneo_detects_changes(self) -> None: hashes = _make_fbneo_hashes() hashes["bios_sets"]["neogeo"]["roms"].append( { "name": "sp-s3.sp1", "size": 131072, "crc32": "91b64be3", } ) with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "fbneo.yml", _make_fbneo_profile()) hashes_path = _write_json(p / "hashes.json", hashes) diff = compute_diff(profile_path, hashes_path, mode="fbneo") self.assertIn("neogeo.zip:sp-s3.sp1", diff["added"]) self.assertIn("neogeo.zip:sp-s2.sp1", diff["updated"]) self.assertEqual(len(diff["removed"]), 0) def test_diff_fbneo_unchanged(self) -> None: profile = _make_fbneo_profile() profile["files"][0]["crc32"] = "9036d879" profile["files"][0]["size"] = 131072 hashes = _make_fbneo_hashes() with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "fbneo.yml", profile) hashes_path = _write_json(p / "hashes.json", hashes) diff = compute_diff(profile_path, hashes_path, mode="fbneo") self.assertEqual(diff["unchanged"], 1) self.assertEqual(len(diff["added"]), 0) self.assertEqual(len(diff["updated"]), 0) def test_diff_fbneo_keeps_a_per_rom_ref(self) -> None: """The scraper knows the driver line; a curated ref knows its ROM.""" profile = _make_fbneo_profile() profile["files"][0]["crc32"] = "9036d879" profile["files"][0]["size"] = 131072 profile["files"][0]["source_ref"] = "src/burn/drv/neogeo/d_neogeo.cpp:1615" with tempfile.TemporaryDirectory() as td: p = Path(td) profile_path = _write_yaml(p / "fbneo.yml", profile) hashes_path = _write_json(p / "hashes.json", _make_fbneo_hashes()) diff = compute_diff(profile_path, hashes_path, mode="fbneo") self.assertEqual(len(diff["updated"]), 0) self.assertEqual(diff["unchanged"], 1) if __name__ == "__main__": unittest.main() class TestSourceRefDrift(unittest.TestCase): """A set whose ROMs are unchanged but whose driver line moved is an update.""" HASHES = { "version": "0.289", "bios_sets": { "neogeo": { "source_file": "src/mame/snk/neogeo.cpp", "source_line": 2500, "roms": [ {"name": "sp-s2.sp1", "size": 131072, "crc32": "9036d879", "sha1": "4f5ed7105b7128794654ce82b51723e16e389543"}, ], } }, } def _profile(self, source_ref): return { "emulator": "MAME", "core_version": "0.287", "files": [ { "name": "neogeo.zip", "category": "bios_zip", "source_ref": source_ref, "contents": [ {"name": "sp-s2.sp1", "size": 131072, "crc32": "9036d879", "sha1": "4f5ed7105b7128794654ce82b51723e16e389543"}, ], } ], } def _diff(self, source_ref): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) prof = _write_yaml(root / "p.yml", self._profile(source_ref)) hashes = root / "h.json" hashes.write_text(json.dumps(self.HASHES), encoding="utf-8") return compute_diff(prof, str(hashes), mode="mame") def test_a_moved_line_counts_as_an_update(self): diff = self._diff("src/mame/snk/neogeo.cpp:2428") self.assertEqual(diff["updated"], ["neogeo"]) self.assertEqual(diff["unchanged"], 0) def test_an_identical_ref_stays_unchanged(self): expected = _build_source_ref(self.HASHES["bios_sets"]["neogeo"]) diff = self._diff(expected) self.assertEqual(diff["updated"], []) self.assertEqual(diff["unchanged"], 1) class TestDerivativeVersion(unittest.TestCase): """Only MAME itself is relabelled with the upstream version.""" HASHES = { "version": "0.289", "bios_sets": { "cdibios": { "source_file": "src/mame/philips/cdi.cpp", "source_line": 484, "roms": [ {"name": "cdi200.rom", "size": 262144, "crc32": "40c4e6b9", "sha1": "d5e9e75bec84e93ea67d0e3f0b8b7e0b6f7d0f5f"}, ], } }, } def _run(self, add_new, version="Git"): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) prof = _write_yaml(root / "p.yml", { "emulator": "SAME CDi", "core_version": version, "files": [{"name": "cdibios.zip", "category": "bios_zip", "source_ref": "old.cpp:1", "contents": []}], }) hashes = root / "h.json" hashes.write_text(json.dumps(self.HASHES), encoding="utf-8") return merge_mame_profile( prof, str(hashes), write=False, add_new=add_new ) def test_a_derivative_keeps_its_own_version(self): self.assertEqual(self._run(add_new=False)["core_version"], "Git") self.assertEqual(self._run(add_new=True)["core_version"], "Git") def test_a_profile_labelled_with_a_mame_release_follows_it(self): merged = self._run(add_new=False, version="0.287") self.assertEqual(merged["core_version"], "0.289") def test_the_ref_is_refreshed_either_way(self): for add_new in (False, True): entry = self._run(add_new)["files"][0] self.assertEqual(entry["source_ref"], "src/mame/philips/cdi.cpp:484")