fix: write the cut md5 rule once

Six places read an md5 shorter than 32 digits, each with its own bound.
digest_verdict accepted any prefix down to one digit, while Batocera
compares the whole string (batocera-systems:1088). A cut value now names
a dump only when one held md5 starts with it (cut_md5_owner, read by the
scraper, the resolver and --from-md5), and every verdict compares whole
digests.

scripts/common.py

@@ -1434,6 +1434,15 @@ def resolve_local_file(
             names_to_try.append(hint_base)
 
     md5_list = parse_md5_list(file_entry.get("md5"))
+    files_db = db.get("files", {})
+    by_md5 = db.get("indexes", {}).get("by_md5", {})
+    # A value cut short stands for the one held md5 it starts, and only
+    # then (cut_md5_owner): the dump it names is still name-guarded below.
+    cut_owners = {
+        cut: owner for cut in md5_list
+        if len(cut) < 32 and (owner := cut_md5_owner(cut, by_md5))
+    }
+    md5_list = [cut_owners.get(value, value) for value in md5_list]
     sha1_candidates = [
         str(value).strip().lower()
         for value in (sha1 if isinstance(sha1, list) else [sha1] if sha1 else [])
@@ -1461,8 +1470,6 @@ def resolve_local_file(
     has_strong_hash = bool(
         sha1_candidates or sha256_candidates or md5_list or crc_candidates
     )
-    files_db = db.get("files", {})
-    by_md5 = db.get("indexes", {}).get("by_md5", {})
     by_name = db.get("indexes", {}).get("by_name", {})
     by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
 
@@ -1483,7 +1490,7 @@ def resolve_local_file(
         # With zipped_file the MD5 identifies the member, not the container.
         if md5_list and not zipped_file:
             actual_md5 = str(entry.get("md5", "")).lower()
-            if not any(actual_md5.startswith(expected) for expected in md5_list):
+            if actual_md5 not in md5_list:
                 return None
             statuses.append("md5_exact")
         if crc_candidates:
@@ -1567,20 +1574,13 @@ def resolve_local_file(
             sha1_match = by_md5.get(md5_candidate)
             if sha1_match and sha1_match in files_db:
                 path = files_db[sha1_match]["path"]
-                # Full MD5 (32 chars) is a strong identifier: trust it
-                # without name guard. Truncated MD5 still needs name check
-                # to avoid cross-contamination.
+                # A full md5 as declared is a strong identifier: trusted
+                # without name guard. One completed from a cut value still
+                # needs the name check to avoid cross-contamination.
                 status = _record_match_status(sha1_match)
                 if status and os.path.exists(path):
-                    if len(md5_candidate) >= 32 or _md5_name_ok(path):
+                    if md5_candidate not in cut_owners.values() or _md5_name_ok(path):
                         return path, status
-            if len(md5_candidate) < 32:
-                for db_md5, db_sha1 in by_md5.items():
-                    if db_md5.startswith(md5_candidate) and db_sha1 in files_db:
-                        path = files_db[db_sha1]["path"]
-                        status = _record_match_status(db_sha1)
-                        if status and os.path.exists(path) and _md5_name_ok(path):
-                            return path, status
 
     # 2b. Path suffix lookup is useful for same-named regional files, but it
     # is identity evidence only when no content hash was declared.  A stale
@@ -3586,6 +3586,7 @@ from hashing import (  # noqa: E402,F401
     md5sum,
     md5_composite,
     parse_md5_list,
+    cut_md5_owner,
     _ALL_ALGORITHMS,
     _md5_composite_cache,
 )

scripts/cross_reference.py

@@ -317,10 +317,7 @@ def _record_carries(sha1: str, record: dict, declared: dict[str, set[str]]) -> b
         **{field: str(record.get(field) or "").lower() for field in ("md5", "sha256", "crc32")},
     }
     return any(
-        actual[field] and (
-            actual[field] in values
-            or (field == "md5" and any(len(v) < 32 and actual[field].startswith(v) for v in values))
-        )
+        actual[field] and actual[field] in values
         for field, values in declared.items()
     )
 

scripts/exporter/candidates.py

@@ -22,11 +22,6 @@ from common import _norm_system_id, resolve_platform_cores, unresolved_platform_
 from exporter.baseline import HASH_FIELDS, NativeSystem, _hash_values
 from launchers import _names as launcher_names
 
-# Batocera cuts some md5 to 29 characters: the prefix still names the dump.
-_FULL_MD5 = 32
-_CUT_MD5 = 29
-
-
 def _tails(path: str) -> list[str]:
     """Every trailing run of segments of a path, longest first."""
     parts = [p for p in path.casefold().strip("/").split("/") if p]
@@ -48,22 +43,12 @@ def _declared_hashes(entry: dict) -> dict[str, set[str]]:
     return {f: values for f, values in found.items() if values}
 
 
-def _same(declared: str, accepted: str) -> bool:
-    """Equal, or a cut md5 that starts the accepted one."""
-    return declared == accepted or (
-        _CUT_MD5 <= len(declared) < _FULL_MD5 and accepted.startswith(declared)
-    )
-
-
 def _accepts(accepted: set[str], declared: dict[str, set[str]]) -> bool:
     """True when the hashes a profile entry accepts, the members of its
     archive included, hold one the platform entry declares."""
     for values in declared.values():
         if values & accepted:
             return True
-        cut = [d for d in values if _CUT_MD5 <= len(d) < _FULL_MD5]
-        if cut and any(a.startswith(d) for d in cut for a in accepted):
-            return True
     return False
 
 
@@ -72,7 +57,7 @@ def _contradicts(profile_entry: dict, declared: dict[str, set[str]]) -> bool:
     the profile entry describes another dump under the same name."""
     for field_name, values in declared.items():
         own = set(_hash_values(profile_entry, field_name))
-        if own and not any(_same(d, a) for d in values for a in own):
+        if own and not set(values) & set(own):
             return True
     return False
 
@@ -158,10 +143,7 @@ class ReadEvidence:
         hashes = {
             v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
         }
-        if hashes & self.hashes or any(
-            len(v) < _FULL_MD5 and any(h.startswith(v) for h in self.hashes)
-            for v in hashes if len(v) >= _CUT_MD5
-        ):
+        if hashes & self.hashes:
             return "hash"
         full = "/" + (tails[0] if tails else "")
         if any("/" + d in full for d in self.directories):
@@ -188,7 +170,7 @@ class ProofIndex:
 
     def __init__(self, readers: list[ReadEvidence]) -> None:
         self._index: dict[str, dict[str, set[int]]] = {
-            kind: {} for kind in ("tail", "launched", "name", "hash", "cut", "dir", "tree", "member")
+            kind: {} for kind in ("tail", "launched", "name", "hash", "dir", "tree", "member")
         }
         for position, evidence in enumerate(readers):
             for kind, keys in (
@@ -199,11 +181,6 @@ class ProofIndex:
             ):
                 for key in keys:
                     self._index[kind].setdefault(key, set()).add(position)
-            # A cut md5 names any hash it is the start of.
-            for value in evidence.hashes:
-                for length in range(_CUT_MD5, _FULL_MD5):
-                    if len(value) >= length:
-                        self._index["cut"].setdefault(value[:length], set()).add(position)
 
     def _hits(self, kind: str, keys: Iterable[str]) -> set[int]:
         table = self._index[kind]
@@ -224,7 +201,6 @@ class ProofIndex:
         hashes = {
             v for f in (*HASH_FIELDS, "alt_md5") for v in _hash_values(entry, f)
         }
-        cut = {v for v in hashes if _CUT_MD5 <= len(v) < _FULL_MD5}
         # "/" + d in "/" + path: d is a run of whole segments ending in "/",
         # so it is a run of the path's segments before the last one.
         parts = tails[0].split("/") if tails else []
@@ -236,7 +212,7 @@ class ProofIndex:
             "path": self._hits("tail", tails[:-1]),
             "launcher": self._hits("launched", names | last),
             "name": self._hits("name", names | last),
-            "hash": self._hits("hash", hashes) | self._hits("cut", cut),
+            "hash": self._hits("hash", hashes),
             "directory": self._hits("dir", runs),
             "data_directory": self._hits("tree", runs)
             | self._index["member"].get(tails[0] if tails else "", set()),
@@ -463,10 +439,6 @@ class EntryCategories:
             for value in values:
                 for position, f in by_hash_index.get(value, ()):
                     found[position] = f
-                if _CUT_MD5 <= len(value) < _FULL_MD5:
-                    for accepted, hits in by_hash_index.items():
-                        if accepted.startswith(value):
-                            found.update(hits)
         if found:
             return [cited(found[p]) for p in sorted(found)]
         named = {position: f for n in names for position, f in by_name_index.get(n, ())}

scripts/hashing.py

@@ -11,6 +11,7 @@ import zipfile
 import zlib
 from pathlib import Path
 from typing import BinaryIO
+from collections.abc import Iterable
 
 
 _ALL_ALGORITHMS = frozenset({"sha1", "md5", "sha256", "crc32", "adler32"})
@@ -130,6 +131,20 @@ def md5_composite_of(source: str | Path | BinaryIO) -> str:
             h.update(zf.read(name))
         return h.hexdigest()
 
+def cut_md5_owner(cut: str, md5s: Iterable[str]) -> str | None:
+    """The one md5 a value cut short starts, None when none or several do.
+
+    The single rule for a cut md5, read by the scraper, the resolver and
+    --from-md5. It names a dump, it never verifies one: Batocera compares
+    the whole string (batocera-systems:1088), so every verdict compares
+    exact digests.
+    """
+    if not cut or len(cut) >= 32:
+        return None
+    found = {md5 for md5 in md5s if md5.startswith(cut)}
+    return found.pop() if len(found) == 1 else None
+
+
 def parse_hash_list(raw: str | list | int | None) -> list[str]:
     """Normalize any hash field into its accepted values, lowercase.
 

scripts/packresolve.py

@@ -14,6 +14,7 @@ import urllib.request
 from pathlib import Path
 
 from largefiles import fetch_large_file
+from hashing import cut_md5_owner
 from common import (
     declared_hash_verdict,
     resolution_is_hash_exact,
@@ -203,8 +204,8 @@ def pending_release_asset(
     return None
 
 
-# Digest length in hex digits -> kind. 29 is the MD5 Batocera writes with
-# its last three digits cut, matched by prefix as the resolver does.
+# Digest length in hex digits -> kind. A 29-digit MD5 is a value cut short,
+# looked up as the resolver looks one up (cut_md5_owner).
 _HASH_KINDS = {8: "crc32", 29: "md5", 32: "md5", 40: "sha1", 64: "sha256"}
 _HEX_DIGITS = re.compile(r"[0-9a-f]+")
 _WORD = re.compile(r"[0-9A-Za-z]+")
@@ -289,9 +290,9 @@ def sha1_for_hash(hash_type: str, hash_val: str, db: dict) -> str | None:
     elif hash_type == "md5":
         by_md5 = indexes.get("by_md5", {})
         sha1 = by_md5.get(hash_val)
-        if sha1 is None and len(hash_val) < 32:
-            prefixed = {v for k, v in by_md5.items() if k.startswith(hash_val)}
-            sha1 = prefixed.pop() if len(prefixed) == 1 else None
+        if sha1 is None:
+            owner = cut_md5_owner(hash_val, by_md5)
+            sha1 = by_md5.get(owner) if owner else None
     else:
         sha1 = next(
             (k for k, e in files_db.items() if e.get("sha256") == hash_val), None

scripts/scraper/batocera_scraper.py

@@ -18,6 +18,7 @@ from pathlib import Path
 
 
 from common import yaml_load
+from hashing import cut_md5_owner
 from safeparse import yaml_load_upstream
 
 from .base_scraper import (
@@ -312,15 +313,15 @@ def _load_md5_index() -> dict[str, str]:
 
 
 def _resolve_truncated_md5(md5: str, md5_index: dict[str, str]) -> str:
-    """Resolve a truncated MD5 to its full 32-char version via prefix match."""
+    """Resolve a truncated MD5 to its full 32-char version (cut_md5_owner)."""
     if not md5 or len(md5) == 32:
         return md5
     if not _MD5_RE.match(md5):
         return md5
-    matches = [k for k in md5_index if k.startswith(md5)]
-    if len(matches) == 1:
-        print(f"  fixed truncated md5: {md5} -> {matches[0]}", file=sys.stderr)
-        return matches[0]
+    owner = cut_md5_owner(md5, md5_index)
+    if owner:
+        print(f"  fixed truncated md5: {md5} -> {owner}", file=sys.stderr)
+        return owner
     return md5
 
 

scripts/validation.py

@@ -857,8 +857,8 @@ def frontend_digest_matches(
 
     resolve_local_file calls a file `hash_mismatch` when any declared hash
     disagrees, but a frontend compares one digest. verify_entry_md5 reads it
-    too: Batocera compares a truncated md5 by prefix and Recalbox hashes a
-    ZIP by its members. Under SIZE_AND_ANY_HASH (RomM) the declared size must
+    too: Recalbox hashes a ZIP by its members. Batocera compares the whole
+    md5 string (batocera-systems:1088), so a value cut short accepts nothing. Under SIZE_AND_ANY_HASH (RomM) the declared size must
     match and then any one of md5, sha1 and crc32 does: a mistyped hash next
     to a right one does not withhold the file, a wrong size does.
     """
@@ -918,9 +918,7 @@ def digest_verdict(
     # Recalbox hashes a .zip by its members only (Bios.cpp:64-68), so the
     # container's own md5 answers nothing there.
     composite_only = file_entry.get("zip_digest") == "composite"
-    if not composite_only and any(
-        actual == d or (len(d) < 32 and actual.startswith(d)) for d in declared
-    ):
+    if not composite_only and actual in declared:
         return True
     member_digest = composite()
     return bool(member_digest) and member_digest.lower() in declared