From ce2417ce0fb4f0f60c25ba22bc1d26b76c476b10 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sun, 18 Oct 2026 14:52:02 +0200 Subject: [PATCH] refactor: read hash lists with one parser --- scripts/cross_reference.py | 8 +++----- scripts/hashing.py | 24 ++++++++++++++++++++---- scripts/profile_sync.py | 8 +++----- scripts/validation.py | 15 +++------------ tests/test_audit_regressions.py | 19 +++++++++++++++++++ 5 files changed, 48 insertions(+), 26 deletions(-) diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 1fc0c729..dbf32475 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -42,6 +42,7 @@ from common import ( standalone_set, select_emulator_profiles, ) +from hashing import parse_hash_list from validation import declaration_settles, outside_gap_scope yaml = require_yaml() @@ -254,9 +255,7 @@ def _declared_content(file_entry: dict) -> dict[str, set[str]]: """The hashes an entry declares, lowercased, by algorithm.""" declared: dict[str, set[str]] = {} for field in _HASH_FIELDS: - raw = file_entry.get(field) - values = raw if isinstance(raw, list) else [raw] - found = {str(v).lower().removeprefix("0x") for v in values if v} + found = set(parse_hash_list(file_entry.get(field))) if found: declared[field] = found return declared @@ -415,8 +414,7 @@ def entry_source(f: dict, index: dict) -> str | None: return source def values(field: str) -> list[str]: - raw = f.get(field) or [] - return [str(v).lower() for v in (raw if isinstance(raw, list) else [raw]) if v] + return parse_hash_list(f.get(field)) held = ( any(index["by_md5"].get(v) for v in parse_md5_list(f.get("md5"))) diff --git a/scripts/hashing.py b/scripts/hashing.py index c06e4dad..c8f8fc7f 100644 --- a/scripts/hashing.py +++ b/scripts/hashing.py @@ -88,13 +88,29 @@ def md5_composite(filepath: str | Path) -> str: _md5_composite_cache[key] = result return result +def parse_hash_list(raw: str | list | int | None) -> list[str]: + """Normalize any hash field into its accepted values, lowercase. + + A field holds one value, a YAML list, or one comma-joined string + (Recalbox md5, mesence's four sha256 for one entry); a crc32 may carry + a 0x prefix. Read one way here, the resolver, the gap report and + profile_sync accept the same values. + """ + if raw is None or raw == "": + return [] + items = raw if isinstance(raw, list) else str(raw).split(",") + out: list[str] = [] + for item in items: + value = str(item).strip().lower().removeprefix("0x") + if value and value not in out: + out.append(value) + return out + + def parse_md5_list(raw: str | list | None) -> list[str]: """Normalize an md5 field into a lowercase list. Platform YAMLs carry Recalbox multi-hash as one comma-separated string, emulator profiles carry a YAML list. Both reach here. """ - if not raw: - return [] - values = raw if isinstance(raw, list) else str(raw).split(",") - return [str(m).strip().lower() for m in values if str(m).strip()] + return parse_hash_list(raw) diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index aa64f2a8..d7cb053f 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -31,6 +31,7 @@ sys.path.insert(0, os.path.dirname(__file__)) import upstream from common import load_emulator_profiles, suggest_names +from hashing import parse_hash_list from upstream import CompareResult, find_renamed DEFAULT_CACHE = upstream.CACHE_DIR @@ -257,11 +258,8 @@ def entry_hashes(entry: dict) -> list[str]: if not isinstance(source, dict): continue for field in HASH_FIELDS: - val = source.get(field) - vals = val if isinstance(val, list) else [val] if val else [] - for v in vals: - v = str(v).lower().removeprefix("0x") - if v and v not in tokens: + for v in parse_hash_list(source.get(field)): + if v not in tokens: tokens.append(v) return tokens diff --git a/scripts/validation.py b/scripts/validation.py index fd03db6f..73174897 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -18,7 +18,7 @@ from common import ( sanitize_pack_path, size_fits, ) -from hashing import md5_composite, parse_md5_list +from hashing import md5_composite, parse_hash_list, parse_md5_list from nativemode import SIZE_AND_ANY_HASH, digest_algorithm, hash_mismatch_excludes_file from ziptools import check_inside_zip @@ -58,18 +58,9 @@ def _entry_rule(emu_name: str, f: dict, checks: set[str]) -> dict: 1 KiB Wii U otp.bin, and the variant search handed Cemu the 3DS file. """ def values(field: str) -> set[str]: - raw = f.get(field) - if not raw or field not in checks: + if field not in checks: return set() - items = raw if isinstance(raw, list) else str(raw).split(",") - out = set() - for item in items: - norm = str(item).strip().lower() - if field == "crc32" and norm.startswith("0x"): - norm = norm[2:] - if norm: - out.add(norm) - return out + return set(parse_hash_list(f.get(field))) raw_size = f.get("size") if "size" in checks else None adler = f.get("known_hash_adler32") or f.get("adler32") diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 6157777f..fc3bd0d3 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -4802,5 +4802,24 @@ class ZippedFileMd5IsParsedOnce(unittest.TestCase): self.assertIn("parse_md5_list(", source, module) + +class AHashListIsReadOneWay(unittest.TestCase): + """mesence declares four sha256 for BS-X.bin as one comma-joined string. + The resolver split it and shipped the file; the gap report and + profile_sync looked the whole string up and called it missing.""" + + def test_every_reader_splits_the_list(self): + from scripts.cross_reference import _declared_content + from scripts.hashing import parse_hash_list + from scripts.profile_sync import entry_hashes + + a, b = "a" * 64, "B" * 64 + entry = {"name": "BS-X.bin", "sha256": f"{a}, {b}", "crc32": "0xDEADBEEF"} + self.assertEqual(parse_hash_list(entry["sha256"]), [a, b.lower()]) + self.assertEqual(_declared_content(entry)["sha256"], {a, b.lower()}) + self.assertEqual(_declared_content(entry)["crc32"], {"deadbeef"}) + self.assertEqual(set(entry_hashes(entry)), {a, b.lower(), "deadbeef"}) + + if __name__ == "__main__": unittest.main()