fix: write the cut md5 rule once

Six places read an md5 shorter than 32 digits, each with its own bound.
digest_verdict accepted any prefix down to one digit, while Batocera
compares the whole string (batocera-systems:1088). A cut value now names
a dump only when one held md5 starts with it (cut_md5_owner, read by the
scraper, the resolver and --from-md5), and every verdict compares whole
digests.

scripts/common.py

@@ -1434,6 +1434,15 @@ def resolve_local_file(
             names_to_try.append(hint_base)
 
     md5_list = parse_md5_list(file_entry.get("md5"))
+    files_db = db.get("files", {})
+    by_md5 = db.get("indexes", {}).get("by_md5", {})
+    # A value cut short stands for the one held md5 it starts, and only
+    # then (cut_md5_owner): the dump it names is still name-guarded below.
+    cut_owners = {
+        cut: owner for cut in md5_list
+        if len(cut) < 32 and (owner := cut_md5_owner(cut, by_md5))
+    }
+    md5_list = [cut_owners.get(value, value) for value in md5_list]
     sha1_candidates = [
         str(value).strip().lower()
         for value in (sha1 if isinstance(sha1, list) else [sha1] if sha1 else [])
@@ -1461,8 +1470,6 @@ def resolve_local_file(
     has_strong_hash = bool(
         sha1_candidates or sha256_candidates or md5_list or crc_candidates
     )
-    files_db = db.get("files", {})
-    by_md5 = db.get("indexes", {}).get("by_md5", {})
     by_name = db.get("indexes", {}).get("by_name", {})
     by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
 
@@ -1483,7 +1490,7 @@ def resolve_local_file(
         # With zipped_file the MD5 identifies the member, not the container.
         if md5_list and not zipped_file:
             actual_md5 = str(entry.get("md5", "")).lower()
-            if not any(actual_md5.startswith(expected) for expected in md5_list):
+            if actual_md5 not in md5_list:
                 return None
             statuses.append("md5_exact")
         if crc_candidates:
@@ -1567,20 +1574,13 @@ def resolve_local_file(
             sha1_match = by_md5.get(md5_candidate)
             if sha1_match and sha1_match in files_db:
                 path = files_db[sha1_match]["path"]
-                # Full MD5 (32 chars) is a strong identifier: trust it
-                # without name guard. Truncated MD5 still needs name check
-                # to avoid cross-contamination.
+                # A full md5 as declared is a strong identifier: trusted
+                # without name guard. One completed from a cut value still
+                # needs the name check to avoid cross-contamination.
                 status = _record_match_status(sha1_match)
                 if status and os.path.exists(path):
-                    if len(md5_candidate) >= 32 or _md5_name_ok(path):
+                    if md5_candidate not in cut_owners.values() or _md5_name_ok(path):
                         return path, status
-            if len(md5_candidate) < 32:
-                for db_md5, db_sha1 in by_md5.items():
-                    if db_md5.startswith(md5_candidate) and db_sha1 in files_db:
-                        path = files_db[db_sha1]["path"]
-                        status = _record_match_status(db_sha1)
-                        if status and os.path.exists(path) and _md5_name_ok(path):
-                            return path, status
 
     # 2b. Path suffix lookup is useful for same-named regional files, but it
     # is identity evidence only when no content hash was declared.  A stale
@@ -3586,6 +3586,7 @@ from hashing import (  # noqa: E402,F401
     md5sum,
     md5_composite,
     parse_md5_list,
+    cut_md5_owner,
     _ALL_ALGORITHMS,
     _md5_composite_cache,
 )

scripts/cross_reference.py

@@ -317,10 +317,7 @@ def _record_carries(sha1: str, record: dict, declared: dict[str, set[str]]) -> b
         **{field: str(record.get(field) or "").lower() for field in ("md5", "sha256", "crc32")},
     }
     return any(
-        actual[field] and (
-            actual[field] in values
-            or (field == "md5" and any(len(v) < 32 and actual[field].startswith(v) for v in values))
-        )
+        actual[field] and actual[field] in values
         for field, values in declared.items()
     )
 

scripts/exporter/candidates.py

@@ -22,11 +22,6 @@ from common import _norm_system_id, resolve_platform_cores, unresolved_platform_
 from exporter.baseline import HASH_FIELDS, NativeSystem, _hash_values
 from launchers import _names as launcher_names
 
-# Batocera cuts some md5 to 29 characters: the prefix still names the dump.
-_FULL_MD5 = 32
-_CUT_MD5 = 29
-
-
 def _tails(path: str) -> list[str]:
     """Every trailing run of segments of a path, longest first."""
     parts = [p for p in path.casefold().strip("/").split("/") if p]
@@ -48,22 +43,12 @@ def _declared_hashes(entry: dict) -> dict[str, set[str]]:
     return {f: values for f, values in found.items() if values}
 
 
-def _same(declared: str, accepted: str) -> bool:
-    """Equal, or a cut md5 that starts the accepted one."""
-    return declared == accepted or (
-        _CUT_MD5 <= len(declared) < _FULL_MD5 and accepted.startswith(declared)
-    )
-
-
 def _accepts(accepted: set[str], declared: dict[str, set[str]]) -> bool:
     """True when the hashes a profile entry accepts, the members of its
     archive included, hold one the platform entry declares."""
     for values in declared.values():
         if values & accepted:
             return True
-        cut = [d for d in values if _CUT_MD5 <= len(d) < _FULL_MD5]
-        if cut and any(a.startswith(d) for d in cut for a in accepted):
-            return True
     return False
 
 
@@ -72,7 +57,7 @@ def _contradicts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
     the profile entry describes another dump under the same name."""
     for field_name, values in declared.items():
         own = set(_hash_values(profile_entry, field_name))
-        if own and not any(_same(d, a) for d in values for a in own):
+        if own and not set(values) & set(own):
             return True
     return False
 
@@ -158,10 +143,7 @@ class ReadEvidence:
         hashes = {
             v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
         }
-        if hashes & self.hashes or any(
-            len(v) < _FULL_MD5 and any(h.startswith(v) for h in self.hashes)
-            for v in hashes if len(v) >= _CUT_MD5
-        ):
+        if hashes & self.hashes:
             return "hash"
         full = "/" + (tails[0] if tails else "")
         if any("/" + d in full for d in self.directories):
@@ -188,7 +170,7 @@ class ProofIndex:
 
     def __init__(self, readers: list[ReadEvidence]) -> None:
         self._index: dict[str, dict[str, set[int]]] = {
-            kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member")
+            kind: {} for kind in ("tail", "launched", "name", "hash", "dir", "tree", "member")
         }
         for position, evidence in enumerate(readers):
             for kind, keys in (
@@ -199,11 +181,6 @@ class ProofIndex:
             ):
                 for key in keys:
                     self._index[kind].setdefault(key, set()).add(position)
-            # A cut md5 names any hash it is the start of.
-            for value in evidence.hashes:
-                for length in range(_CUT_MD5, _FULL_MD5):
-                    if len(value) >= length:
-                        self._index["cut"].setdefault(value[:length], set()).add(position)
 
     def _hits(self, kind: str, keys: Iterable[str]) -> set[int]:
         table = self._index[kind]
@@ -224,7 +201,6 @@ class ProofIndex:
         hashes = {
             v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
         }
-        cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5}
         # "/" + d in "/" + path: d is a run of whole segments ending in "/",
         # so it is a run of the path's segments before the last one.
         parts = tails[0].split("/") if tails else []
@@ -236,7 +212,7 @@ class ProofIndex:
             "path": self._hits("tail", tails[:-1]),
             "launcher": self._hits("launched", names | last),
             "name": self._hits("name", names | last),
-            "hash": self._hits("hash", hashes) | self._hits("cut", cut),
+            "hash": self._hits("hash", hashes),
             "directory": self._hits("dir", runs),
             "data_directory": self._hits("tree", runs)
             | self._index["member"].get(tails[0] if tails else "", set()),
@@ -463,10 +439,6 @@ class EntryCategories:
             for value in values:
                 for position, f in by_hash_index.get(value, ()):
                     found[position] = f
-                if _CUT_MD5 <= len(value) < _FULL_MD5:
-                    for accepted, hits in by_hash_index.items():
-                        if accepted.startswith(value):
-                            found.update(hits)
         if found:
             return [cited(found[p]) for p in sorted(found)]
         named = {position: f for n in names for position, f in by_name_index.get(n, ())}

scripts/hashing.py

@@ -11,6 +11,7 @@ import zipfile
 import zlib
 from pathlib import Path
 from typing import BinaryIO
+from collections.abc import Iterable
 
 
 _ALL_ALGORITHMS = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"})
@@ -130,6 +131,20 @@ def md5_composite_of(source: str | Path | BinaryIO) -> str:
             h.update(zf.read(name))
         return h.hexdigest()
 
+def cut_md5_owner(cut: str, md5s: Iterable[str]) -> str | None:
+    """The one md5 a value cut short starts, None when none or several do.
+
+    The single rule for a cut md5, read by the scraper, the resolver and
+    --from-md5. It names a dump, it never verifies one: Batocera compares
+    the whole string (batocera-systems:1088), so every verdict compares
+    exact digests.
+    """
+    if not cut or len(cut) >= 32:
+        return None
+    found = {md5 for md5 in md5s if md5.startswith(cut)}
+    return found.pop() if len(found) == 1 else None
+
+
 def parse_hash_list(raw: str | list | int | None) -> list[str]:
     """Normalize any hash field into its accepted values, lowercase.
 

scripts/packresolve.py

@@ -14,6 +14,7 @@ import urllib.request
 from pathlib import Path
 
 from largefiles import fetch_large_file
+from hashing import cut_md5_owner
 from common import (
     declared_hash_verdict,
     resolution_is_hash_exact,
@@ -203,8 +204,8 @@ def pending_release_asset(
     return None
 
 
-# Digest length in hex digits -> kind. 29 is the MD5 Batocera writes with
-# its last three digits cut, matched by prefix as the resolver does.
+# Digest length in hex digits -> kind. A 29-digit MD5 is a value cut short,
+# looked up as the resolver looks one up (cut_md5_owner).
 _HASH_KINDS = {8: "crc32", 29: "md5", 32: "md5", 40: "sha1", 64: "sha256"}
 _HEX_DIGITS = re.compile(r"[0-9a-f]+")
 _WORD = re.compile(r"[0-9A-Za-z]+")
@@ -289,9 +290,9 @@ def sha1_for_hash(hash_type: str, hash_val: str, db: dict) -> str | None:
     elif hash_type == "md5":
         by_md5 = indexes.get("by_md5", {})
         sha1 = by_md5.get(hash_val)
-        if sha1 is None and len(hash_val) < 32:
-            prefixed = {v for k, v in by_md5.items() if k.startswith(hash_val)}
-            sha1 = prefixed.pop() if len(prefixed) == 1 else None
+        if sha1 is None:
+            owner = cut_md5_owner(hash_val, by_md5)
+            sha1 = by_md5.get(owner) if owner else None
     else:
         sha1 = next(
             (k for k, e in files_db.items() if e.get("sha256") == hash_val), None

scripts/scraper/batocera_scraper.py

@@ -18,6 +18,7 @@ from pathlib import Path
 
 
 from common import yaml_load
+from hashing import cut_md5_owner
 from safeparse import yaml_load_upstream
 
 from .base_scraper import (
@@ -312,15 +313,15 @@ def _load_md5_index() -> dict[str, str]:
 
 
 def _resolve_truncated_md5(md5: str, md5_index: dict[str, str]) -> str:
-    """Resolve a truncated MD5 to its full 32-char version via prefix match."""
+    """Resolve a truncated MD5 to its full 32-char version (cut_md5_owner)."""
     if not md5 or len(md5) == 32:
         return md5
     if not _MD5_RE.match(md5):
         return md5
-    matches = [k for k in md5_index if k.startswith(md5)]
-    if len(matches) == 1:
-        print(f"  fixed truncated md5: {md5} -> {matches[0]}", file=sys.stderr)
-        return matches[0]
+    owner = cut_md5_owner(md5, md5_index)
+    if owner:
+        print(f"  fixed truncated md5: {md5} -> {owner}", file=sys.stderr)
+        return owner
     return md5
 
 

scripts/validation.py

@@ -857,8 +857,8 @@ def frontend_digest_matches(
 
     resolve_local_file calls a file `hash_mismatch` when any declared hash
     disagrees, but a frontend compares one digest. verify_entry_md5 reads it
-    too: Batocera compares a truncated md5 by prefix and Recalbox hashes a
-    ZIP by its members. Under SIZE_AND_ANY_HASH (RomM) the declared size must
+    too: Recalbox hashes a ZIP by its members. Batocera compares the whole
+    md5 string (batocera-systems:1088), so a value cut short accepts nothing. Under SIZE_AND_ANY_HASH (RomM) the declared size must
     match and then any one of md5, sha1 and crc32 does: a mistyped hash next
     to a right one does not withhold the file, a wrong size does.
     """
@@ -918,9 +918,7 @@ def digest_verdict(
     # Recalbox hashes a .zip by its members only (Bios.cpp:64-68), so the
     # container's own md5 answers nothing there.
     composite_only = file_entry.get("zip_digest") == "composite"
-    if not composite_only and any(
-        actual == d or (len(d) < 32 and actual.startswith(d)) for d in declared
-    ):
+    if not composite_only and actual in declared:
         return True
     member_digest = composite()
     return bool(member_digest) and member_digest.lower() in declared

tests/test_audit_regressions.py

@@ -1300,23 +1300,29 @@ class OneBaselineVerdict(unittest.TestCase):
             )
 
     def test_the_builder_accepts_what_verify_accepts(self):
-        """verify took a 29-character md5 by prefix and a ZIP by its
-        composite, but the builder compared exact md5s and refused both when
-        another declared hash disagreed."""
+        """verify took a ZIP by its composite, but the builder compared exact
+        md5s and refused it when another declared hash disagreed. A value cut
+        short accepts nothing: Batocera compares the whole md5 string
+        (batocera-systems:1088), and verify accepted any prefix, down to one
+        digit."""
         import hashlib
 
         from scripts.common import md5_composite
-        from scripts.validation import baseline_verdict
+        from scripts.validation import baseline_verdict, digest_verdict
 
         with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory:
             loose = Path(directory) / "abc.bin"
             loose.write_bytes(b"abc")
-            truncated = {"name": "abc.bin", "md5": hashlib.md5(b"abc").hexdigest()[:29],
-                         "crc32": "deadbeef"}
-            self.assertEqual(
-                baseline_verdict(truncated, str(loose), "hash_mismatch", "md5"),
-                ("frontend_digest_exact", None),
+            md5 = hashlib.md5(b"abc").hexdigest()
+            truncated = {"name": "abc.bin", "md5": md5[:29], "crc32": "deadbeef"}
+            self.assertIsNotNone(
+                baseline_verdict(truncated, str(loose), "hash_mismatch", "md5")[1]
             )
+            for length in (1, 29, 31):
+                self.assertFalse(
+                    digest_verdict({"md5": md5[:length]}, 3, lambda: {"md5": md5}, "md5")
+                )
+            self.assertTrue(digest_verdict({"md5": md5}, 3, lambda: {"md5": md5}, "md5"))
             archive = self._archive(Path(directory), "rom.bin")
             composite = {"name": "dragon32.zip", "md5": md5_composite(archive),
                          "crc32": "deadbeef"}

wiki/adding-a-scraper.md

@@ -232,8 +232,8 @@ System ID consistency matters for cross-platform operations. The same console mu
   are available.
 - An MD5 shorter than 32 characters is completed when exactly one MD5 in
   `database.json` starts with it, as the Batocera scraper does
-  (`_resolve_truncated_md5`), and kept as written otherwise. The resolution layer
-  matches what remains by prefix.
+  (`_resolve_truncated_md5`, through `cut_md5_owner`), and kept as written
+  otherwise. A value kept short verifies nothing.
 
 Scraped data reflects what the upstream declares, which can differ from what the emulator source reads. The scraper's job is faithful transcription of upstream data, not correction. Corrections happen in the emulator profiles (source-verified) and in `_shared.yml` (curated). If a scraper meets a hash or filename that differs from the emulator source, log a warning and still include the upstream value. The divergence will surface during truth diffing.
 

wiki/verification-modes.md

@@ -88,8 +88,10 @@ Batocera's script compares the full string ([`batocera-systems:1088`](https://gi
 shorter than 32 characters matches no file there. The Batocera scraper completes such
 a value when exactly one MD5 in `database.json` starts with it
 (`_resolve_truncated_md5` in `scripts/scraper/batocera_scraper.py`) and keeps it as
-written otherwise. `resolve_local_file` matches a value still shorter than 32
-characters by prefix, and only for a file whose name the entry also names.
+written otherwise. `resolve_local_file` and `--from-md5` read a value still shorter
+than 32 characters by the same rule (`cut_md5_owner` in `scripts/hashing.py`), and the
+resolver keeps the file only when its name is one the entry names. Every verdict
+compares whole digests, as Batocera does: a cut value never verifies a file.
 
 ### md5_composite (Recalbox ZIP verification)