diff --git a/scripts/scraper/_hash_merge.py b/scripts/scraper/_hash_merge.py index c26f3cc0..d635f4b4 100644 --- a/scripts/scraper/_hash_merge.py +++ b/scripts/scraper/_hash_merge.py @@ -564,12 +564,63 @@ def _format_contents(contents: list[dict]) -> str: return "\n".join(lines) +# Content values an FBNeo entry carries, in the order a new one is written. +_FBNEO_VALUES = ("size", "crc32", "sha1") +_ENTRY_START = re.compile(r"^ - name: ", re.M) +_FIELD_LINE = re.compile(r"^ (?P[a-z0-9_]+):\s*(?P[^#\n]*?)\s*(?P#.*)?$") + + +def _patch_fbneo_values(text: str, changed: dict[tuple[str, str], dict]) -> str: + """Rewrite size/crc32/sha1 inside the blocks of the changed entries.""" + starts = [m.start() for m in _ENTRY_START.finditer(text)] + [len(text)] + out = [text[: starts[0]]] + for begin, end in zip(starts, starts[1:]): + block = text[begin:end] + fields = {} + for line in block.splitlines()[1:]: + match = _FIELD_LINE.match(line) + if match: + fields[match["key"]] = match["value"].strip().strip('"') + name = block.splitlines()[0].removeprefix(" - name:").strip().strip('"') + entry = changed.get((fields.get("archive", ""), name)) + if entry is not None: + block = _patch_block_values(block, entry) + out.append(block) + return "".join(out) + + +def _patch_block_values(block: str, entry: dict) -> str: + lines = block.split("\n") + for key in _FBNEO_VALUES: + value = entry.get(key) + if not value: + continue + rendered = str(value) if key == "size" else f'"{value}"' + for i, line in enumerate(lines): + match = _FIELD_LINE.match(line) + if match and match["key"] == key: + if match["value"].strip().strip('"') != str(value): + # A size comment spells the old value; it goes with it. + lines[i] = f" {key}: {rendered}" + break + else: + anchor = max( + (i for i, line in enumerate(lines) + if (m := _FIELD_LINE.match(line)) and m["key"] in ("archive", *_FBNEO_VALUES)), + default=0, + ) + lines.insert(anchor + 1, f" {key}: {rendered}") + return "\n".join(lines) + + def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None: """Write merged FBNeo profile using text-based patching. FBNeo profiles have individual ROM entries with archive: field. - Only patches core_version and appends new ROM entries. - Existing entries are left untouched (CRC32 changes are rare). + Patches core_version, the content values of existing entries that + changed, and appends new ROM entries. Advancing core_version alone left + a profile claiming a release whose crc32 it did not carry; comments and + curated source_refs stay as written. """ p = Path(path) backup = p.with_suffix(PROFILE_BACKUP_SUFFIX) @@ -580,11 +631,25 @@ def _backup_and_write_fbneo(path: str, data: dict, hashes: dict) -> None: # Identify new ROM entries by comparing parsed data keys, not text search existing_data = yaml_load(original) or {} - existing_keys = { - (f["archive"], f["name"]) + existing_by_key = { + (f["archive"], f["name"]): f for f in existing_data.get("files", []) if f.get("archive") } + existing_keys = set(existing_by_key) + changed = { + (f["archive"], f["name"]): f + for f in data.get("files", []) + if f.get("archive") + and (f["archive"], f["name"]) in existing_by_key + and any( + str(f.get(k) or "") != str(existing_by_key[(f["archive"], f["name"])].get(k) or "") + for k in _FBNEO_VALUES + if f.get(k) + ) + } + if changed: + patched = _patch_fbneo_values(patched, changed) new_roms = [ f for f in data.get("files", []) diff --git a/tests/test_hash_merge.py b/tests/test_hash_merge.py index c75ccf35..c3807759 100644 --- a/tests/test_hash_merge.py +++ b/tests/test_hash_merge.py @@ -706,5 +706,58 @@ class AnUnreadableCloneStopsTheScrape(unittest.TestCase): scraper._get_commit() + +class WritersPutTheValuesOnDisk(unittest.TestCase): + """No test read back what the writers wrote. The FBNeo writer advanced + core_version and never patched a changed crc32, and a broken MAME + patcher would have done the same without a failing test. These + fixtures use the layout the profiles use (entries indented two).""" + + def setUp(self): + self.dir = Path(tempfile.mkdtemp(dir=Path(__file__).resolve().parents[1] / "tmp")) + self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, True)) + + def test_fbneo_writes_a_changed_crc32(self): + profile = self.dir / "fbneo.yml" + profile.write_text( + 'emulator: FBNeo\ncore_version: "v1.0.0.02"\nfiles:\n' + ' - name: "sp-s2.sp1"\n archive: neogeo.zip\n required: false\n' + ' size: 131072 # 0x20000\n crc32: "00000000"\n' + ' note: "MVS Asia/Europe ver. 5"\n source_ref: "src/burn/drv/neogeo/d_neogeo.cpp:1625"\n', + encoding="utf-8", + ) + hashes = _write_json(self.dir / "h.json", { + "version": "v1.0.0.03", + "bios_sets": {"neogeo": {"source_file": "src/burn/drv/neogeo/d_neogeo.cpp", + "source_line": 1625, + "roms": [{"name": "sp-s2.sp1", "size": 131072, + "crc32": "9036d879"}]}}, + }) + merge_fbneo_profile(str(profile), hashes, write=True, add_new=False) + written = yaml.safe_load(profile.read_text(encoding="utf-8")) + entry = written["files"][0] + self.assertEqual(written["core_version"], "v1.0.0.03") + self.assertEqual(entry["crc32"], "9036d879") + self.assertEqual(entry["note"], "MVS Asia/Europe ver. 5") + self.assertEqual(entry["source_ref"], "src/burn/drv/neogeo/d_neogeo.cpp:1625") + self.assertIn("size: 131072 # 0x20000", profile.read_text(encoding="utf-8")) + + def test_mame_writes_member_values(self): + profile = self.dir / "mame.yml" + profile.write_text( + 'emulator: MAME\ncore_version: "0.285"\nfiles:\n' + ' - name: neogeo.zip\n required: true\n category: bios_zip\n' + ' system: snk-neogeo-mvs\n source_ref: "src/mame/neogeo/neogeo.cpp:2400"\n' + ' contents:\n - name: sp-s2.sp1\n size: 131072\n' + ' crc32: oldcrc32\n description: Europe MVS (Ver. 2)\n', + encoding="utf-8", + ) + hashes = _write_json(self.dir / "h.json", _make_mame_hashes()) + merge_mame_profile(str(profile), hashes, write=True) + written = yaml.safe_load(profile.read_text(encoding="utf-8")) + member = written["files"][0]["contents"][0] + self.assertEqual(member["crc32"], "9036d879") + + if __name__ == "__main__": unittest.main()