From 5a3d9bdc9d3bb2f8c704013497a48ffe38055cc3 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:20:00 +0200 Subject: [PATCH] fix: resolve the DSP ROMs as a pair the code takes Dolphin and its forks hash dsp_rom.bin and dsp_coef.bin together and warn on any combination outside known_roms. The profiles now carry the accepted pairs, and the resolver keeps both files inside the first pair the collection holds whole: MMJR and Ishiiruka were served the free ROM beside the official coefficients. --- emulators/dolphin-mmjr.yml | 4 ++ emulators/dolphin-mmjr2.yml | 4 ++ emulators/dolphin.yml | 4 ++ emulators/ishiiruka.yml | 4 ++ emulators/primehack.yml | 4 ++ emulators/triforce.yml | 4 ++ schemas/emulator.schema.json | 15 ++++++ scripts/common.py | 56 ++++++++++++++++++++-- tests/test_adler32_resolution.py | 79 ++++++++++++++++++++++++++++++++ wiki/profiling.md | 1 + 10 files changed, 170 insertions(+), 5 deletions(-) diff --git a/emulators/dolphin-mmjr.yml b/emulators/dolphin-mmjr.yml index eadb6aae..7789f9f0 100644 --- a/emulators/dolphin-mmjr.yml +++ b/emulators/dolphin-mmjr.yml @@ -144,6 +144,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0x66f334fe", "0x128ea7a2"] adler32_byteswap: true + adler32_pairs_with: "dsp_coef.bin" + adler32_pairs: [["0x66f334fe", "0xf3b93527"], ["0x128ea7a2", "0xa4a575f5"]] description: "DSP instruction ROM" note: "DSP LLE only, and LLE aborts when the file is absent or not 8192 bytes. The adler32 is taken over the byte-swapped image and the value here is the Nintendo pair; five replacement pairs are also accepted." source_ref: "Source/Core/Common/CommonPaths.h:108, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:120-152, Source/Core/Core/DSP/DSPCore.h:35, Source/Core/Core/DSP/DSPCore.cpp:38-80" @@ -156,6 +158,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0xf3b93527", "0xa4a575f5"] adler32_byteswap: true + adler32_pairs_with: "dsp_rom.bin" + adler32_pairs: [["0xf3b93527", "0x66f334fe"], ["0xa4a575f5", "0x128ea7a2"]] description: "DSP coefficient ROM" note: "Read by DSP LLE alongside the instruction ROM, and by the AX ucode under HLE for polyphase resampling, which drops any file that is not 0x1000 bytes and plays on without it." source_ref: "Source/Core/Common/CommonPaths.h:109, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:120-152, Source/Core/Core/HW/DSPHLE/UCodes/AX.cpp:45-70, Source/Core/Core/DSP/DSPCore.h:49, Source/Core/Core/DSP/DSPCore.cpp:38-105" diff --git a/emulators/dolphin-mmjr2.yml b/emulators/dolphin-mmjr2.yml index 70525438..9691c5f4 100644 --- a/emulators/dolphin-mmjr2.yml +++ b/emulators/dolphin-mmjr2.yml @@ -180,6 +180,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0x66f334fe", "0xe789b5a5"] adler32_byteswap: true + adler32_pairs_with: "dsp_coef.bin" + adler32_pairs: [["0x66f334fe", "0xf3b93527"], ["0xe789b5a5", "0xa4a575f5"]] description: "DSP instruction ROM" note: "DSP LLE only, and a file that is absent or not 8192 bytes fails initialisation, which aborts the emulation thread. The adler32 is taken over the byte-swapped image and the value here is the Nintendo pair; six replacement pairs are also accepted." source_ref: "Source/Core/Common/CommonPaths.h:120, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-105, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:107-120, Source/Core/Core/DSP/DSPCore.h:44, Source/Core/Core/DSP/DSPCore.cpp:28-71" @@ -192,6 +194,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0xf3b93527", "0xa4a575f5"] adler32_byteswap: true + adler32_pairs_with: "dsp_rom.bin" + adler32_pairs: [["0xf3b93527", "0x66f334fe"], ["0xa4a575f5", "0xe789b5a5"]] description: "DSP coefficient ROM" note: "Read by DSP LLE alongside the instruction ROM, and by the AX ucode under HLE for polyphase resampling, which drops any file that is not 0x1000 bytes and plays on without it." source_ref: "Source/Core/Common/CommonPaths.h:121, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-120, Source/Core/Core/HW/DSPHLE/UCodes/AX.cpp:42-69, Source/Core/Core/DSP/DSPCore.h:58, Source/Core/Core/DSP/DSPCore.cpp:28-102" diff --git a/emulators/dolphin.yml b/emulators/dolphin.yml index 1ba1d8c7..da1e02f5 100644 --- a/emulators/dolphin.yml +++ b/emulators/dolphin.yml @@ -78,6 +78,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0x66f334fe", "0xe789b5a5"] adler32_byteswap: true + adler32_pairs_with: "dsp_coef.bin" + adler32_pairs: [["0x66f334fe", "0xf3b93527"], ["0xe789b5a5", "0xa4a575f5"]] note: "DSP instruction ROM for LLE audio. Free replacement (v0.4) included" source_ref: "Source/Core/Common/CommonPaths.h:136, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:84-117, Source/Core/Core/DSP/DSPCore.cpp:28-102" @@ -90,6 +92,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0xf3b93527", "0xa4a575f5"] adler32_byteswap: true + adler32_pairs_with: "dsp_rom.bin" + adler32_pairs: [["0xf3b93527", "0x66f334fe"], ["0xa4a575f5", "0xe789b5a5"]] note: "DSP coefficient ROM for LLE audio and HLE polyphase resampling. Free replacement included" source_ref: "Source/Core/Common/CommonPaths.h:137, Source/Core/Core/DSP/DSPCore.cpp:32-33, Source/Core/Core/HW/DSPHLE/UCodes/AX.cpp:55-62, Source/Core/Core/DSP/DSPCore.cpp:28-102" diff --git a/emulators/ishiiruka.yml b/emulators/ishiiruka.yml index c83a7033..4c060709 100644 --- a/emulators/ishiiruka.yml +++ b/emulators/ishiiruka.yml @@ -74,6 +74,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0x66f334fe", "0x128ea7a2"] adler32_byteswap: true + adler32_pairs_with: "dsp_coef.bin" + adler32_pairs: [["0x66f334fe", "0xf3b93527"], ["0x128ea7a2", "0xa4a575f5"]] note: "DSP instruction ROM for LLE audio. Free replacement included" source_ref: "Source/Core/Common/CommonPaths.h:112, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:142-150, Source/Core/Core/DSP/DSPCore.cpp:48,67, Source/Core/Core/DSP/DSPCore.cpp:38-105" @@ -86,6 +88,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0xf3b93527", "0xa4a575f5"] adler32_byteswap: true + adler32_pairs_with: "dsp_rom.bin" + adler32_pairs: [["0xf3b93527", "0x66f334fe"], ["0xa4a575f5", "0x128ea7a2"]] note: "DSP coefficient ROM for LLE audio and HLE polyphase resampling. Free replacement included" source_ref: "Source/Core/Common/CommonPaths.h:113, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:143-153, Source/Core/Core/HW/DSPHLE/UCodes/AX.cpp:43-72, Source/Core/Core/DSP/DSPCore.cpp:48,68, Source/Core/Core/DSP/DSPCore.cpp:38-105" diff --git a/emulators/primehack.yml b/emulators/primehack.yml index 98db24f9..33ae0178 100644 --- a/emulators/primehack.yml +++ b/emulators/primehack.yml @@ -71,6 +71,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0x66f334fe", "0xe789b5a5"] adler32_byteswap: true + adler32_pairs_with: "dsp_coef.bin" + adler32_pairs: [["0x66f334fe", "0xf3b93527"], ["0xe789b5a5", "0xa4a575f5"]] note: "DSP instruction ROM for LLE audio. Free replacement (v0.4) included" source_ref: "Source/Core/Common/CommonPaths.h:135, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-117, Source/Core/Core/DSP/DSPCore.cpp:28-102" @@ -82,6 +84,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0xf3b93527", "0xa4a575f5"] adler32_byteswap: true + adler32_pairs_with: "dsp_rom.bin" + adler32_pairs: [["0xf3b93527", "0x66f334fe"], ["0xa4a575f5", "0xe789b5a5"]] note: "DSP coefficient ROM for LLE audio. Free replacement included" source_ref: "Source/Core/Common/CommonPaths.h:138, Source/Core/Core/DSP/DSPCore.cpp:32-38, Source/Core/Core/DSP/DSPCore.cpp:28-102" diff --git a/emulators/triforce.yml b/emulators/triforce.yml index 33c1d2fe..ddfa19ec 100644 --- a/emulators/triforce.yml +++ b/emulators/triforce.yml @@ -230,6 +230,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0x66f334fe", "0xe789b5a5"] adler32_byteswap: true + adler32_pairs_with: "dsp_coef.bin" + adler32_pairs: [["0x66f334fe", "0xf3b93527"], ["0xe789b5a5", "0xa4a575f5"]] required: false hle_fallback: true source_ref: "Source/Core/Common/CommonPaths.h:145, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-105,109-118, Source/Core/Core/DSP/DSPCore.h:44, Source/Core/Core/DSP/DSPCore.cpp:36-38, Source/Core/Core/DSP/DSPCore.cpp:28-102" @@ -248,6 +250,8 @@ files: validation: [size, adler32] known_hash_adler32: ["0xf3b93527", "0xa4a575f5"] adler32_byteswap: true + adler32_pairs_with: "dsp_rom.bin" + adler32_pairs: [["0xf3b93527", "0x66f334fe"], ["0xa4a575f5", "0xe789b5a5"]] required: false hle_fallback: true source_ref: "Source/Core/Common/CommonPaths.h:146, Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-105,110-119, Source/Core/Core/DSP/DSPCore.h:58, Source/Core/Core/HW/DSPHLE/UCodes/AX.cpp:61-62, Source/Core/Core/DSP/DSPCore.cpp:28-102" diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index e9baf222..26a45828 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -424,6 +424,21 @@ "adler32_byteswap": { "type": "boolean" }, + "adler32_pairs_with": { + "description": "The file hashed together with this one; adler32_pairs lists the combinations the code accepts.", + "type": "string" + }, + "adler32_pairs": { + "description": "Accepted Adler-32 combinations, this file's value first, in the code's table order.", + "type": "array", + "items": { + "type": "array", + "items": {"type": "string"}, + "minItems": 2, + "maxItems": 2 + }, + "minItems": 1 + }, "agnostic": { "type": "boolean" }, diff --git a/scripts/common.py b/scripts/common.py index a05845f0..bd7ec210 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -92,13 +92,55 @@ def adler32_of(path: str, byteswap: bool = False) -> str: return format(zlib.adler32(data) & 0xFFFFFFFF, "08x") -def adler32_fits(file_entry: dict, path: str) -> bool: +def _adler32_value(value: object) -> str: + return str(value).lower().removeprefix("0x") + + +def _held_adler32(name: str, db: dict, byteswap: bool) -> set[str]: + """The Adler-32 of every copy of a name the collection holds.""" + files_db = db.get("files", {}) + held: set[str] = set() + for sha1 in db.get("indexes", {}).get("by_name", {}).get(name, []): + path = files_db.get(sha1, {}).get("path", "") + try: + held.add(adler32_of(path, byteswap)) + except OSError: + continue + return held + + +def held_adler32_pair(file_entry: dict, db: dict) -> str | None: + """The entry's own value in the first pair its code accepts that is held. + + Dolphin hashes dsp_rom.bin and dsp_coef.bin together and warns on any + combination outside its table: two files each accepted alone can still + be a pair it refuses. ``adler32_pairs`` lists the accepted pairs, the + entry's own value first, in the code's table order; both members walk + the same pairs over the same collection, so they settle on one pair. + None when the entry declares no pair or none is held whole. + """ + pairs = file_entry.get("adler32_pairs") + partner = file_entry.get("adler32_pairs_with") + if not pairs or not partner: + return None + byteswap = bool(file_entry.get("adler32_byteswap")) + own = _held_adler32(file_entry.get("name", ""), db, byteswap) + other = _held_adler32(partner, db, byteswap) + for mine, theirs in pairs: + if _adler32_value(mine) in own and _adler32_value(theirs) in other: + return _adler32_value(mine) + return None + + +def adler32_fits(file_entry: dict, path: str, db: dict | None = None) -> bool: """Whether a candidate carries an Adler-32 the entry's code accepts. Dolphin hashes its DSP ROMs and warns on every pair it does not take as current: the v0.3.1 free dsp_rom.bin is answered by its path, and shipped beside the official dsp_coef.bin it is a pair Dolphin rejects. A value declared without ``validation: [adler32]`` rejects nothing. + With the collection at hand, a candidate outside the held pair the + code accepts (``held_adler32_pair``) is refused too. """ validation = file_entry.get("validation") if isinstance(validation, dict): @@ -107,11 +149,15 @@ def adler32_fits(file_entry: dict, path: str) -> bool: if "adler32" not in (validation or []) or not declared: return True values = declared if isinstance(declared, list) else [declared] - accepted = {str(v).lower().removeprefix("0x") for v in values} + accepted = {_adler32_value(v) for v in values} try: - return adler32_of(path, bool(file_entry.get("adler32_byteswap"))) in accepted + actual = adler32_of(path, bool(file_entry.get("adler32_byteswap"))) except OSError: return False + if actual not in accepted: + return False + chosen = held_adler32_pair(file_entry, db) if db is not None else None + return chosen is None or actual == chosen def size_fits(file_entry: dict, candidate_size: int) -> bool: @@ -1115,7 +1161,7 @@ def resolve_local_file( if not os.path.exists(path): continue if not has_strong_hash: - if not adler32_fits(file_entry, path): + if not adler32_fits(file_entry, path, db): continue return path, "path_exact" status = _record_match_status(match_sha1) @@ -1126,7 +1172,7 @@ def resolve_local_file( def _size_ok(match_sha1: str) -> bool: record = files_db.get(match_sha1, {}) return name_match_size_ok(file_entry, record.get("size")) and adler32_fits( - file_entry, record.get("path", "") + file_entry, record.get("path", ""), db ) # An entry the profile documents as unsourceable is one the collection diff --git a/tests/test_adler32_resolution.py b/tests/test_adler32_resolution.py index 235291a0..5c6f2f50 100644 --- a/tests/test_adler32_resolution.py +++ b/tests/test_adler32_resolution.py @@ -64,6 +64,57 @@ class TheAcceptedChecksumWins(unittest.TestCase): format(zlib.adler32(b"\x02\x01" * 8), "08x")) +class TheTwoRomsFormAPairTheCodeTakes(unittest.TestCase): + """Dolphin-MMJR accepts the v0.3.1 free dsp_rom.bin (128ea7a2) with the + free coef, and the official ROM with the official coef. The collection + holds the free ROM at the path, the official one in .variants/ and only + the official coef: each passed alone, and the pair served was free ROM + plus official coef, which its known_roms table refuses.""" + + def setUp(self): + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + held = { + ("dsp_rom.bin", root / "bios/Sys/GC/dsp_rom.bin"): b"\x01\x02" * 8, + ("dsp_rom.bin", root / "bios/.variants/dsp_rom.bin.off"): b"\x03\x04" * 8, + ("dsp_coef.bin", root / "bios/.variants/dsp_coef.bin.off"): b"\x05\x06" * 4, + } + files, by_name, by_suffix = {}, {}, {} + for (name, path), data in held.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + files[sha1] = {"path": str(path), "name": name, "sha1": sha1, "size": len(data)} + by_name.setdefault(name, []).append(sha1) + if "Sys/GC" in str(path): + by_suffix["GC/dsp_rom.bin"] = [sha1] + self.db = {"files": files, "indexes": {"by_name": by_name, "by_md5": {}, + "by_path_suffix": by_suffix, + "by_crc32": {}}} + free_rom = adler32_of(str(root / "bios/Sys/GC/dsp_rom.bin"), True) + self.official_rom = root / "bios/.variants/dsp_rom.bin.off" + official_rom = adler32_of(str(self.official_rom), True) + official_coef = adler32_of(str(root / "bios/.variants/dsp_coef.bin.off"), True) + free_coef = "a4a575f5" + self.rom = { + "name": "dsp_rom.bin", "path": "GC/dsp_rom.bin", "size": 16, + "validation": ["size", "adler32"], "adler32_byteswap": True, + "known_hash_adler32": [f"0x{official_rom}", f"0x{free_rom}"], + "adler32_pairs_with": "dsp_coef.bin", + "adler32_pairs": [[f"0x{official_rom}", f"0x{official_coef}"], + [f"0x{free_rom}", f"0x{free_coef}"]], + } + + def test_the_rom_follows_the_held_coef(self): + path, status = resolve_local_file(self.rom, self.db, dest_hint="GC/dsp_rom.bin") + self.assertEqual((path, status), (str(self.official_rom), "name_exact")) + + def test_without_a_whole_pair_each_value_stands_alone(self): + rom = {**self.rom, "adler32_pairs": [["0x00000001", "0x00000002"]]} + path, status = resolve_local_file(rom, self.db, dest_hint="GC/dsp_rom.bin") + self.assertEqual(status, "path_exact") + + class DolphinGetsAPairItTakes(unittest.TestCase): def test_each_dolphin_dsp_rom_resolves_to_an_accepted_dump(self): from common import load_emulator_profiles # noqa: PLC0415 @@ -85,5 +136,33 @@ class DolphinGetsAPairItTakes(unittest.TestCase): self.assertIn(adler32_of(path, True), accepted) + def test_every_paired_profile_resolves_a_pair_its_code_takes(self): + from common import load_emulator_profiles # noqa: PLC0415 + + database = REPO_ROOT / "database.json" + if not database.exists(): + self.skipTest("database.json is not generated") + db = json.loads(database.read_text()) + profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + for emu, profile in sorted(profiles.items()): + entries = {e["name"]: e for e in profile.get("files", []) + if isinstance(e, dict) and e.get("adler32_pairs")} + for name, entry in entries.items(): + partner = entries.get(entry["adler32_pairs_with"]) + if partner is None: + continue + with self.subTest(emulator=emu, name=name): + values = [] + for e in (entry, partner): + path, _status = resolve_local_file( + {**e, "source_profile": emu}, db, + dest_hint=e.get("path") or "", + ) + self.assertIsNotNone(path) + values.append(adler32_of(path, bool(e.get("adler32_byteswap")))) + pairs = {tuple(v.lower().removeprefix("0x") for v in pair) + for pair in entry["adler32_pairs"]} + self.assertIn(tuple(values), pairs) + if __name__ == "__main__": unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index b5fe28f4..0eeda1f6 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -353,6 +353,7 @@ which CI validates every profile against. | `min_size`, `max_size` | size range when the code accepts a range | | `md5`, `sha1`, `crc32`, `sha256` | expected hashes from source code | | `known_hash_adler32` | expected Adler-32 hash (Dolphin's DSP ROMs); pair with `adler32_byteswap` when the code hashes 16-bit byte-swapped data | +| `adler32_pairs_with`, `adler32_pairs` | the file hashed together with this one and the combinations the code accepts, this file's value first, in table order: Dolphin warns on a DSP ROM pair outside its `known_roms` even when each file passes alone | | `pin_frozen` | why `source_commit` is held on purpose rather than followed. The profile is then judged against its own revision, like one pinned to a superseded tag | | `upstream_gone` | why the declared upstream can no longer be reached, when that is settled rather than transient. Records the death instead of leaving the profile failing every pass; profile_sync reports it as declared, and reports the contradiction if the forge answers again | | `source_mirror` | a repository carrying the same tree, consulted after `source` and `upstream`. What keeps a profile checkable once its own forge stops answering |