diff --git a/scripts/common.py b/scripts/common.py index 62be6e5cf..920079775 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1434,6 +1434,15 @@ def resolve_local_file( names_to_try.append(hint_base) md5_list = parse_md5_list(file_entry.get("md5")) + files_db = db.get("files", {}) + by_md5 = db.get("indexes", {}).get("by_md5", {}) + # A value cut short stands for the one held md5 it starts, and only + # then (cut_md5_owner): the dump it names is still name-guarded below. + cut_owners = { + cut: owner for cut in md5_list + if len(cut) < 32 and (owner := cut_md5_owner(cut, by_md5)) + } + md5_list = [cut_owners.get(value, value) for value in md5_list] sha1_candidates = [ str(value).strip().lower() for value in (sha1 if isinstance(sha1, list) else [sha1] if sha1 else []) @@ -1461,8 +1470,6 @@ def resolve_local_file( has_strong_hash = bool( sha1_candidates or sha256_candidates or md5_list or crc_candidates ) - files_db = db.get("files", {}) - by_md5 = db.get("indexes", {}).get("by_md5", {}) by_name = db.get("indexes", {}).get("by_name", {}) by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {}) @@ -1483,7 +1490,7 @@ def resolve_local_file( # With zipped_file the MD5 identifies the member, not the container. if md5_list and not zipped_file: actual_md5 = str(entry.get("md5", "")).lower() - if not any(actual_md5.startswith(expected) for expected in md5_list): + if actual_md5 not in md5_list: return None statuses.append("md5_exact") if crc_candidates: @@ -1567,20 +1574,13 @@ def resolve_local_file( sha1_match = by_md5.get(md5_candidate) if sha1_match and sha1_match in files_db: path = files_db[sha1_match]["path"] - # Full MD5 (32 chars) is a strong identifier: trust it - # without name guard. Truncated MD5 still needs name check - # to avoid cross-contamination. + # A full md5 as declared is a strong identifier: trusted + # without name guard. One completed from a cut value still + # needs the name check to avoid cross-contamination. status = _record_match_status(sha1_match) if status and os.path.exists(path): - if len(md5_candidate) >= 32 or _md5_name_ok(path): + if md5_candidate not in cut_owners.values() or _md5_name_ok(path): return path, status - if len(md5_candidate) < 32: - for db_md5, db_sha1 in by_md5.items(): - if db_md5.startswith(md5_candidate) and db_sha1 in files_db: - path = files_db[db_sha1]["path"] - status = _record_match_status(db_sha1) - if status and os.path.exists(path) and _md5_name_ok(path): - return path, status # 2b. Path suffix lookup is useful for same-named regional files, but it # is identity evidence only when no content hash was declared. A stale @@ -3586,6 +3586,7 @@ from hashing import ( # noqa: E402,F401 md5sum, md5_composite, parse_md5_list, + cut_md5_owner, _ALL_ALGORITHMS, _md5_composite_cache, ) diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index f92545b27..7aff523f0 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -317,10 +317,7 @@ def _record_carries(sha1: str, record: dict, declared: dict[str, set[str]]) -> b **{field: str(record.get(field) or "").lower() for field in ("md5", "sha256", "crc32")}, } return any( - actual[field] and ( - actual[field] in values - or (field == "md5" and any(len(v) < 32 and actual[field].startswith(v) for v in values)) - ) + actual[field] and actual[field] in values for field, values in declared.items() ) diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py index d2e259928..579ee3b4a 100644 --- a/scripts/exporter/candidates.py +++ b/scripts/exporter/candidates.py @@ -22,11 +22,6 @@ from common import _norm_system_id, resolve_platform_cores, unresolved_platform_ from exporter.baseline import HASH_FIELDS, NativeSystem, _hash_values from launchers import _names as launcher_names -# Batocera cuts some md5 to 29 characters: the prefix still names the dump. -_FULL_MD5 = 32 -_CUT_MD5 = 29 - - def _tails(path: str) -> list[str]: """Every trailing run of segments of a path, longest first.""" parts = [p for p in path.casefold().strip("/").split("/") if p] @@ -48,22 +43,12 @@ def _declared_hashes(entry: dict) -> dict[str, set[str]]: return {f: values for f, values in found.items() if values} -def _same(declared: str, accepted: str) -> bool: - """Equal, or a cut md5 that starts the accepted one.""" - return declared == accepted or ( - _CUT_MD5 <= len(declared) < _FULL_MD5 and accepted.startswith(declared) - ) - - def _accepts(accepted: set[str], declared: dict[str, set[str]]) -> bool: """True when the hashes a profile entry accepts, the members of its archive included, hold one the platform entry declares.""" for values in declared.values(): if values & accepted: return True - cut = [d for d in values if _CUT_MD5 <= len(d) < _FULL_MD5] - if cut and any(a.startswith(d) for d in cut for a in accepted): - return True return False @@ -72,7 +57,7 @@ def _contradicts(profile_entry: dict, declared: dict[str, set[str]]) -> bool: the profile entry describes another dump under the same name.""" for field_name, values in declared.items(): own = set(_hash_values(profile_entry, field_name)) - if own and not any(_same(d, a) for d in values for a in own): + if own and not set(values) & set(own): return True return False @@ -158,10 +143,7 @@ class ReadEvidence: hashes = { v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f) } - if hashes & self.hashes or any( - len(v) < _FULL_MD5 and any(h.startswith(v) for h in self.hashes) - for v in hashes if len(v) >= _CUT_MD5 - ): + if hashes & self.hashes: return "hash" full = "/" + (tails[0] if tails else "") if any("/" + d in full for d in self.directories): @@ -188,7 +170,7 @@ class ProofIndex: def __init__(self, readers: list[ReadEvidence]) -> None: self._index: dict[str, dict[str, set[int]]] = { - kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member") + kind: {} for kind in ("tail", "launched", "name", "hash", "dir", "tree", "member") } for position, evidence in enumerate(readers): for kind, keys in ( @@ -199,11 +181,6 @@ class ProofIndex: ): for key in keys: self._index[kind].setdefault(key, set()).add(position) - # A cut md5 names any hash it is the start of. - for value in evidence.hashes: - for length in range(_CUT_MD5, _FULL_MD5): - if len(value) >= length: - self._index["cut"].setdefault(value[:length], set()).add(position) def _hits(self, kind: str, keys: Iterable[str]) -> set[int]: table = self._index[kind] @@ -224,7 +201,6 @@ class ProofIndex: hashes = { v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f) } - cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5} # "/" + d in "/" + path: d is a run of whole segments ending in "/", # so it is a run of the path's segments before the last one. parts = tails[0].split("/") if tails else [] @@ -236,7 +212,7 @@ class ProofIndex: "path": self._hits("tail", tails[:-1]), "launcher": self._hits("launched", names | last), "name": self._hits("name", names | last), - "hash": self._hits("hash", hashes) | self._hits("cut", cut), + "hash": self._hits("hash", hashes), "directory": self._hits("dir", runs), "data_directory": self._hits("tree", runs) | self._index["member"].get(tails[0] if tails else "", set()), @@ -463,10 +439,6 @@ class EntryCategories: for value in values: for position, f in by_hash_index.get(value, ()): found[position] = f - if _CUT_MD5 <= len(value) < _FULL_MD5: - for accepted, hits in by_hash_index.items(): - if accepted.startswith(value): - found.update(hits) if found: return [cited(found[p]) for p in sorted(found)] named = {position: f for n in names for position, f in by_name_index.get(n, ())} diff --git a/scripts/hashing.py b/scripts/hashing.py index 2dee62464..5383b002b 100644 --- a/scripts/hashing.py +++ b/scripts/hashing.py @@ -11,6 +11,7 @@ import zipfile import zlib from pathlib import Path from typing import BinaryIO +from collections.abc import Iterable _ALL_ALGORITHMS = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"}) @@ -130,6 +131,20 @@ def md5_composite_of(source: str | Path | BinaryIO) -> str: h.update(zf.read(name)) return h.hexdigest() +def cut_md5_owner(cut: str, md5s: Iterable[str]) -> str | None: + """The one md5 a value cut short starts, None when none or several do. + + The single rule for a cut md5, read by the scraper, the resolver and + --from-md5. It names a dump, it never verifies one: Batocera compares + the whole string (batocera-systems:1088), so every verdict compares + exact digests. + """ + if not cut or len(cut) >= 32: + return None + found = {md5 for md5 in md5s if md5.startswith(cut)} + return found.pop() if len(found) == 1 else None + + def parse_hash_list(raw: str | list | int | None) -> list[str]: """Normalize any hash field into its accepted values, lowercase. diff --git a/scripts/packresolve.py b/scripts/packresolve.py index d21ec1f16..45552b46a 100644 --- a/scripts/packresolve.py +++ b/scripts/packresolve.py @@ -14,6 +14,7 @@ import urllib.request from pathlib import Path from largefiles import fetch_large_file +from hashing import cut_md5_owner from common import ( declared_hash_verdict, resolution_is_hash_exact, @@ -203,8 +204,8 @@ def pending_release_asset( return None -# Digest length in hex digits -> kind. 29 is the MD5 Batocera writes with -# its last three digits cut, matched by prefix as the resolver does. +# Digest length in hex digits -> kind. A 29-digit MD5 is a value cut short, +# looked up as the resolver looks one up (cut_md5_owner). _HASH_KINDS = {8: "crc32", 29: "md5", 32: "md5", 40: "sha1", 64: "sha256"} _HEX_DIGITS = re.compile(r"[0-9a-f]+") _WORD = re.compile(r"[0-9A-Za-z]+") @@ -289,9 +290,9 @@ def sha1_for_hash(hash_type: str, hash_val: str, db: dict) -> str | None: elif hash_type == "md5": by_md5 = indexes.get("by_md5", {}) sha1 = by_md5.get(hash_val) - if sha1 is None and len(hash_val) < 32: - prefixed = {v for k, v in by_md5.items() if k.startswith(hash_val)} - sha1 = prefixed.pop() if len(prefixed) == 1 else None + if sha1 is None: + owner = cut_md5_owner(hash_val, by_md5) + sha1 = by_md5.get(owner) if owner else None else: sha1 = next( (k for k, e in files_db.items() if e.get("sha256") == hash_val), None diff --git a/scripts/scraper/batocera_scraper.py b/scripts/scraper/batocera_scraper.py index dde7798ae..f1e3fcdcc 100644 --- a/scripts/scraper/batocera_scraper.py +++ b/scripts/scraper/batocera_scraper.py @@ -18,6 +18,7 @@ from pathlib import Path from common import yaml_load +from hashing import cut_md5_owner from safeparse import yaml_load_upstream from .base_scraper import ( @@ -312,15 +313,15 @@ def _load_md5_index() -> dict[str, str]: def _resolve_truncated_md5(md5: str, md5_index: dict[str, str]) -> str: - """Resolve a truncated MD5 to its full 32-char version via prefix match.""" + """Resolve a truncated MD5 to its full 32-char version (cut_md5_owner).""" if not md5 or len(md5) == 32: return md5 if not _MD5_RE.match(md5): return md5 - matches = [k for k in md5_index if k.startswith(md5)] - if len(matches) == 1: - print(f" fixed truncated md5: {md5} -> {matches[0]}", file=sys.stderr) - return matches[0] + owner = cut_md5_owner(md5, md5_index) + if owner: + print(f" fixed truncated md5: {md5} -> {owner}", file=sys.stderr) + return owner return md5 diff --git a/scripts/validation.py b/scripts/validation.py index 5a4272dd5..f8f93819b 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -857,8 +857,8 @@ def frontend_digest_matches( resolve_local_file calls a file `hash_mismatch` when any declared hash disagrees, but a frontend compares one digest. verify_entry_md5 reads it - too: Batocera compares a truncated md5 by prefix and Recalbox hashes a - ZIP by its members. Under SIZE_AND_ANY_HASH (RomM) the declared size must + too: Recalbox hashes a ZIP by its members. Batocera compares the whole + md5 string (batocera-systems:1088), so a value cut short accepts nothing. Under SIZE_AND_ANY_HASH (RomM) the declared size must match and then any one of md5, sha1 and crc32 does: a mistyped hash next to a right one does not withhold the file, a wrong size does. """ @@ -918,9 +918,7 @@ def digest_verdict( # Recalbox hashes a .zip by its members only (Bios.cpp:64-68), so the # container's own md5 answers nothing there. composite_only = file_entry.get("zip_digest") == "composite" - if not composite_only and any( - actual == d or (len(d) < 32 and actual.startswith(d)) for d in declared - ): + if not composite_only and actual in declared: return True member_digest = composite() return bool(member_digest) and member_digest.lower() in declared diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 284c3e730..ba0915394 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -1300,23 +1300,29 @@ class OneBaselineVerdict(unittest.TestCase): ) def test_the_builder_accepts_what_verify_accepts(self): - """verify took a 29-character md5 by prefix and a ZIP by its - composite, but the builder compared exact md5s and refused both when - another declared hash disagreed.""" + """verify took a ZIP by its composite, but the builder compared exact + md5s and refused it when another declared hash disagreed. A value cut + short accepts nothing: Batocera compares the whole md5 string + (batocera-systems:1088), and verify accepted any prefix, down to one + digit.""" import hashlib from scripts.common import md5_composite - from scripts.validation import baseline_verdict + from scripts.validation import baseline_verdict, digest_verdict with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory: loose = Path(directory) / "abc.bin" loose.write_bytes(b"abc") - truncated = {"name": "abc.bin", "md5": hashlib.md5(b"abc").hexdigest()[:29], - "crc32": "deadbeef"} - self.assertEqual( - baseline_verdict(truncated, str(loose), "hash_mismatch", "md5"), - ("frontend_digest_exact", None), + md5 = hashlib.md5(b"abc").hexdigest() + truncated = {"name": "abc.bin", "md5": md5[:29], "crc32": "deadbeef"} + self.assertIsNotNone( + baseline_verdict(truncated, str(loose), "hash_mismatch", "md5")[1] ) + for length in (1, 29, 31): + self.assertFalse( + digest_verdict({"md5": md5[:length]}, 3, lambda: {"md5": md5}, "md5") + ) + self.assertTrue(digest_verdict({"md5": md5}, 3, lambda: {"md5": md5}, "md5")) archive = self._archive(Path(directory), "rom.bin") composite = {"name": "dragon32.zip", "md5": md5_composite(archive), "crc32": "deadbeef"} diff --git a/wiki/adding-a-scraper.md b/wiki/adding-a-scraper.md index a8ea434ef..6f10d2b46 100644 --- a/wiki/adding-a-scraper.md +++ b/wiki/adding-a-scraper.md @@ -232,8 +232,8 @@ System ID consistency matters for cross-platform operations. The same console mu are available. - An MD5 shorter than 32 characters is completed when exactly one MD5 in `database.json` starts with it, as the Batocera scraper does - (`_resolve_truncated_md5`), and kept as written otherwise. The resolution layer - matches what remains by prefix. + (`_resolve_truncated_md5`, through `cut_md5_owner`), and kept as written + otherwise. A value kept short verifies nothing. Scraped data reflects what the upstream declares, which can differ from what the emulator source reads. The scraper's job is faithful transcription of upstream data, not correction. Corrections happen in the emulator profiles (source-verified) and in `_shared.yml` (curated). If a scraper meets a hash or filename that differs from the emulator source, log a warning and still include the upstream value. The divergence will surface during truth diffing. diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index 744fdc61c..286165b7c 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -88,8 +88,10 @@ Batocera's script compares the full string ([`batocera-systems:1088`](https://gi shorter than 32 characters matches no file there. The Batocera scraper completes such a value when exactly one MD5 in `database.json` starts with it (`_resolve_truncated_md5` in `scripts/scraper/batocera_scraper.py`) and keeps it as -written otherwise. `resolve_local_file` matches a value still shorter than 32 -characters by prefix, and only for a file whose name the entry also names. +written otherwise. `resolve_local_file` and `--from-md5` read a value still shorter +than 32 characters by the same rule (`cut_md5_owner` in `scripts/hashing.py`), and the +resolver keeps the file only when its name is one the entry names. Every verdict +compares whole digests, as Batocera does: a cut value never verifies a file. ### md5_composite (Recalbox ZIP verification)