refactor: read hash lists with one parser

scripts/cross_reference.py

@@ -42,6 +42,7 @@ from common import (
     standalone_set,
     select_emulator_profiles,
 )
+from hashing import parse_hash_list
 from validation import declaration_settles, outside_gap_scope
 
 yaml = require_yaml()
@@ -254,9 +255,7 @@ def _declared_content(file_entry: dict) -> dict[str, set[str]]:
     """The hashes an entry declares, lowercased, by algorithm."""
     declared: dict[str, set[str]] = {}
     for field in _HASH_FIELDS:
-        raw = file_entry.get(field)
-        values = raw if isinstance(raw, list) else [raw]
-        found = {str(v).lower().removeprefix("0x") for v in values if v}
+        found = set(parse_hash_list(file_entry.get(field)))
         if found:
             declared[field] = found
     return declared
@@ -415,8 +414,7 @@ def entry_source(f: dict, index: dict) -> str | None:
             return source
 
     def values(field: str) -> list[str]:
-        raw = f.get(field) or []
-        return [str(v).lower() for v in (raw if isinstance(raw, list) else [raw]) if v]
+        return parse_hash_list(f.get(field))
 
     held = (
         any(index["by_md5"].get(v) for v in parse_md5_list(f.get("md5")))

scripts/hashing.py

@@ -88,13 +88,29 @@ def md5_composite(filepath: str | Path) -> str:
     _md5_composite_cache[key] = result
     return result
 
+def parse_hash_list(raw: str | list | int | None) -> list[str]:
+    """Normalize any hash field into its accepted values, lowercase.
+
+    A field holds one value, a YAML list, or one comma-joined string
+    (Recalbox md5, mesence's four sha256 for one entry); a crc32 may carry
+    a 0x prefix. Read one way here, the resolver, the gap report and
+    profile_sync accept the same values.
+    """
+    if raw is None or raw == "":
+        return []
+    items = raw if isinstance(raw, list) else str(raw).split(",")
+    out: list[str] = []
+    for item in items:
+        value = str(item).strip().lower().removeprefix("0x")
+        if value and value not in out:
+            out.append(value)
+    return out
+
+
 def parse_md5_list(raw: str | list | None) -> list[str]:
     """Normalize an md5 field into a lowercase list.
 
     Platform YAMLs carry Recalbox multi-hash as one comma-separated string,
     emulator profiles carry a YAML list. Both reach here.
     """
-    if not raw:
-        return []
-    values = raw if isinstance(raw, list) else str(raw).split(",")
-    return [str(m).strip().lower() for m in values if str(m).strip()]
+    return parse_hash_list(raw)

scripts/profile_sync.py

@@ -31,6 +31,7 @@ sys.path.insert(0, os.path.dirname(__file__))
 
 import upstream
 from common import load_emulator_profiles, suggest_names
+from hashing import parse_hash_list
 from upstream import CompareResult, find_renamed
 
 DEFAULT_CACHE = upstream.CACHE_DIR
@@ -257,11 +258,8 @@ def entry_hashes(entry: dict) -> list[str]:
         if not isinstance(source, dict):
             continue
         for field in HASH_FIELDS:
-            val = source.get(field)
-            vals = val if isinstance(val, list) else [val] if val else []
-            for v in vals:
-                v = str(v).lower().removeprefix("0x")
-                if v and v not in tokens:
+            for v in parse_hash_list(source.get(field)):
+                if v not in tokens:
                     tokens.append(v)
     return tokens
 

scripts/validation.py

@@ -18,7 +18,7 @@ from common import (
     sanitize_pack_path,
     size_fits,
 )
-from hashing import md5_composite, parse_md5_list
+from hashing import md5_composite, parse_hash_list, parse_md5_list
 from nativemode import SIZE_AND_ANY_HASH, digest_algorithm, hash_mismatch_excludes_file
 from ziptools import check_inside_zip
 
@@ -58,18 +58,9 @@ def _entry_rule(emu_name: str, f: dict, checks: set[str]) -> dict:
     1 KiB Wii U otp.bin, and the variant search handed Cemu the 3DS file.
     """
     def values(field: str) -> set[str]:
-        raw = f.get(field)
-        if not raw or field not in checks:
+        if field not in checks:
             return set()
-        items = raw if isinstance(raw, list) else str(raw).split(",")
-        out = set()
-        for item in items:
-            norm = str(item).strip().lower()
-            if field == "crc32" and norm.startswith("0x"):
-                norm = norm[2:]
-            if norm:
-                out.add(norm)
-        return out
+        return set(parse_hash_list(f.get(field)))
 
     raw_size = f.get("size") if "size" in checks else None
     adler = f.get("known_hash_adler32") or f.get("adler32")

tests/test_audit_regressions.py

@@ -4802,5 +4802,24 @@ class ZippedFileMd5IsParsedOnce(unittest.TestCase):
             self.assertIn("parse_md5_list(", source, module)
 
 
+
+class AHashListIsReadOneWay(unittest.TestCase):
+    """mesence declares four sha256 for BS-X.bin as one comma-joined string.
+    The resolver split it and shipped the file; the gap report and
+    profile_sync looked the whole string up and called it missing."""
+
+    def test_every_reader_splits_the_list(self):
+        from scripts.cross_reference import _declared_content
+        from scripts.hashing import parse_hash_list
+        from scripts.profile_sync import entry_hashes
+
+        a, b = "a" * 64, "B" * 64
+        entry = {"name": "BS-X.bin", "sha256": f"{a}, {b}", "crc32": "0xDEADBEEF"}
+        self.assertEqual(parse_hash_list(entry["sha256"]), [a, b.lower()])
+        self.assertEqual(_declared_content(entry)["sha256"], {a, b.lower()})
+        self.assertEqual(_declared_content(entry)["crc32"], {"deadbeef"})
+        self.assertEqual(set(entry_hashes(entry)), {a, b.lower(), "deadbeef"})
+
+
 if __name__ == "__main__":
     unittest.main()