diff --git a/scripts/common.py b/scripts/common.py index 7b4c9a2b..a2ba4cb4 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -73,6 +73,47 @@ def name_match_size_ok(file_entry: dict, candidate_size: int | None) -> bool: return size_fits(file_entry, candidate_size) +def adler32_of(path: str, byteswap: bool = False) -> str: + """The Adler-32 of a file, over its 16-bit words swapped when asked. + + Dolphin swaps every 16-bit word of its DSP ROMs before hashing them + (Common::swap16 in DSPLLE.cpp:LoadDSPRom). + """ + import struct # noqa: PLC0415 + import zlib # noqa: PLC0415 + + with open(path, "rb") as f: + data = f.read() + if byteswap: + if len(data) % 2: + data += b"\x00" + count = len(data) // 2 + data = struct.pack(f">{count}H", *struct.unpack(f"<{count}H", data)) + return format(zlib.adler32(data) & 0xFFFFFFFF, "08x") + + +def adler32_fits(file_entry: dict, path: str) -> bool: + """Whether a candidate carries an Adler-32 the entry's code accepts. + + Dolphin hashes its DSP ROMs and warns on every pair it does not take + as current: the v0.3.1 free dsp_rom.bin is answered by its path, and + shipped beside the official dsp_coef.bin it is a pair Dolphin rejects. + A value declared without ``validation: [adler32]`` rejects nothing. + """ + validation = file_entry.get("validation") + if isinstance(validation, dict): + validation = validation.get("core", []) + declared = file_entry.get("known_hash_adler32") + if "adler32" not in (validation or []) or not declared: + return True + values = declared if isinstance(declared, list) else [declared] + accepted = {str(v).lower().removeprefix("0x") for v in values} + try: + return adler32_of(path, bool(file_entry.get("adler32_byteswap"))) in accepted + except OSError: + return False + + def size_fits(file_entry: dict, candidate_size: int) -> bool: """Whether a size is one the entry declares. @@ -950,6 +991,8 @@ def resolve_local_file( if not os.path.exists(path): continue if not has_strong_hash: + if not adler32_fits(file_entry, path): + continue return path, "path_exact" status = _record_match_status(match_sha1) if status: @@ -957,8 +1000,9 @@ def resolve_local_file( # 3. No MD5 = any file with that name or alias (existence check) def _size_ok(match_sha1: str) -> bool: - return name_match_size_ok( - file_entry, files_db.get(match_sha1, {}).get("size") + record = files_db.get(match_sha1, {}) + return name_match_size_ok(file_entry, record.get("size")) and adler32_fits( + file_entry, record.get("path", "") ) # An entry the profile documents as unsourceable is one the collection diff --git a/scripts/validation.py b/scripts/validation.py index f6e7761d..6cd3d532 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -10,7 +10,13 @@ from __future__ import annotations import os from collections.abc import Mapping -from common import compute_hashes, name_match_size_ok, sanitize_pack_path, size_fits +from common import ( + adler32_of, + compute_hashes, + name_match_size_ok, + sanitize_pack_path, + size_fits, +) from hashing import parse_md5_list from nativemode import digest_algorithm, hash_mismatch_excludes_file from ziptools import check_inside_zip @@ -21,22 +27,9 @@ _CRYPTO_CHECKS = frozenset({"signature", "crypto"}) def _adler32_byteswapped(path: str) -> str: - """Compute adler32 on 16-bit byte-swapped data. + """Adler-32 over 16-bit byte-swapped data, as Dolphin hashes its DSP ROMs.""" + return adler32_of(path, byteswap=True) - Dolphin's DSP loader swaps every 16-bit word before hashing - (Common::swap16 in DSPLLE.cpp:LoadDSPRom). This reproduces that - transform so verify.py can match the expected adler32 values. - """ - import struct - import zlib - - with open(path, "rb") as f: - data = f.read() - # Pad to even length if necessary - if len(data) % 2: - data += b"\x00" - swapped = struct.pack(f">{len(data) // 2}H", *struct.unpack(f"<{len(data) // 2}H", data)) - return format(zlib.adler32(swapped) & 0xFFFFFFFF, "08x") # All reproducible validation types. _HASH_CHECKS = frozenset({"crc32", "md5", "sha1", "adler32"}) diff --git a/tests/test_adler32_resolution.py b/tests/test_adler32_resolution.py new file mode 100644 index 00000000..235291a0 --- /dev/null +++ b/tests/test_adler32_resolution.py @@ -0,0 +1,89 @@ +"""A file answered by its path or name must carry a checksum its core takes. + +Dolphin hashes its DSP ROMs with Adler-32 and warns on every pair older +than the v0.4 free ROM. The collection stores the v0.3.1 dsp_rom.bin at the +path Dolphin reads, so the resolver answered it there: shipped next to the +official dsp_coef.bin, it made a pair Dolphin rejects. The pack builder +corrected it later, every other reader of the resolution did not. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import sys +import tempfile +import unittest +import zlib +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from common import adler32_of, resolve_local_file # noqa: E402 + + +class TheAcceptedChecksumWins(unittest.TestCase): + def setUp(self): + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + self.old = root / "bios/Sys/GC/dsp_rom.bin" + self.good = root / "bios/.variants/dsp_rom.bin.good" + files, by_name, by_suffix = {}, {}, {} + for path, data in ((self.old, b"\x01\x02" * 8), (self.good, b"\x03\x04" * 8)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + files[sha1] = {"path": str(path), "name": "dsp_rom.bin", "sha1": sha1, + "size": len(data)} + by_name.setdefault("dsp_rom.bin", []).append(sha1) + by_suffix["GC/dsp_rom.bin"] = [hashlib.sha1(self.old.read_bytes()).hexdigest()] + self.db = {"files": files, "indexes": {"by_name": by_name, "by_md5": {}, + "by_path_suffix": by_suffix, + "by_crc32": {}}} + self.entry = { + "name": "dsp_rom.bin", "path": "GC/dsp_rom.bin", "size": 16, + "validation": ["size", "adler32"], "adler32_byteswap": True, + "known_hash_adler32": [f"0x{adler32_of(str(self.good), True)}"], + } + + def test_the_path_gives_way_to_the_accepted_file(self): + path, status = resolve_local_file(self.entry, self.db, dest_hint="GC/dsp_rom.bin") + self.assertEqual((path, status), (str(self.good), "name_exact")) + + def test_an_informative_value_rejects_nothing(self): + entry = {**self.entry, "validation": ["size"]} + path, status = resolve_local_file(entry, self.db, dest_hint="GC/dsp_rom.bin") + self.assertEqual((path, status), (str(self.old), "path_exact")) + + def test_the_swap_is_on_16_bit_words(self): + self.assertEqual(adler32_of(str(self.old)), + format(zlib.adler32(b"\x01\x02" * 8), "08x")) + self.assertEqual(adler32_of(str(self.old), True), + format(zlib.adler32(b"\x02\x01" * 8), "08x")) + + +class DolphinGetsAPairItTakes(unittest.TestCase): + def test_each_dolphin_dsp_rom_resolves_to_an_accepted_dump(self): + from common import load_emulator_profiles # noqa: PLC0415 + + database = REPO_ROOT / "database.json" + if not database.exists(): + self.skipTest("database.json is not generated") + db = json.loads(database.read_text()) + profile = load_emulator_profiles(str(REPO_ROOT / "emulators"))["dolphin"] + for entry in profile["files"]: + if entry["name"] not in ("dsp_rom.bin", "dsp_coef.bin"): + continue + with self.subTest(name=entry["name"]): + path, _status = resolve_local_file( + {**entry, "source_profile": "dolphin"}, db, dest_hint=entry["path"] + ) + self.assertIsNotNone(path) + accepted = {v.lower().removeprefix("0x") for v in entry["known_hash_adler32"]} + self.assertIn(adler32_of(path, True), accepted) + + +if __name__ == "__main__": + unittest.main()