From a160a5bc6bb217e4024c38f1ae9b63e620beef68 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 05:20:59 +0200 Subject: [PATCH] fix: name the hash formats --from-md5 accepts A value of another length was looked up as an MD5 and reported NOT FOUND, a SHA-256 was dropped, and an input with no hash ended on "No valid hashes found in input". Digests are now read by length (CRC32, MD5 and its 29-digit Batocera form, SHA-1, SHA-256), SHA-256 is looked up in the database, and any other value is refused with the formats accepted. --- scripts/generate_pack.py | 42 ++++++------ scripts/packresolve.py | 116 ++++++++++++++++++++++---------- tests/test_audit_regressions.py | 39 +++++++++++ wiki/tools.md | 12 ++-- 4 files changed, 152 insertions(+), 57 deletions(-) diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index cf209d3e..a752b3f1 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -2081,8 +2081,6 @@ def generate_md5_pack( ) -> str | None: """Build a pack from an explicit list of hashes with layout context.""" files_db = db.get("files", {}) - by_md5 = db.get("indexes", {}).get("by_md5", {}) - by_crc32 = db.get("indexes", {}).get("by_crc32", {}) if zip_contents is None: zip_contents = {} @@ -2143,15 +2141,8 @@ def generate_md5_pack( with _writing_pack(zip_path) as zf: for hash_type, hash_val in hashes: - sha1 = None - if hash_type == "sha1" and hash_val in files_db: - sha1 = hash_val - elif hash_type == "md5": - sha1 = by_md5.get(hash_val) - elif hash_type == "crc32": - sha1 = by_crc32.get(hash_val) - - if not sha1 or sha1 not in files_db: + sha1 = sha1_for_hash(hash_type, hash_val, db) + if not sha1: not_in_db.append(hash_val) continue @@ -3146,9 +3137,13 @@ def _main(held: contextlib.ExitStack) -> None: help="List available targets for the platform", ) parser.add_argument( - "--from-md5", help="Hash(es) to look up or pack (comma-separated)" + "--from-md5", + help="Hash(es) to look up or pack, comma-separated: CRC32, MD5, SHA-1 or SHA-256", + ) + parser.add_argument( + "--from-md5-file", + help="File with hashes, one per line, alone or beside a file name as md5sum writes it", ) - parser.add_argument("--from-md5-file", help="File with hashes (one per line)") parser.add_argument( "--manifest", action="store_true", @@ -3223,13 +3218,20 @@ def _main(held: contextlib.ExitStack) -> None: hold_data_dirs(held, load_data_dir_registry(args.platforms_dir)) has_from_md5 = bool(args.from_md5 or getattr(args, "from_md5_file", None)) if has_from_md5: - hashes = ( - parse_hash_input(args.from_md5) - if args.from_md5 - else parse_hash_file(args.from_md5_file) - ) + try: + hashes = ( + parse_hash_input(args.from_md5) + if args.from_md5 + else parse_hash_file(args.from_md5_file) + ) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) if not hashes: - print("No valid hashes found in input", file=sys.stderr) + print( + f"ERROR: no hash in the input. Accepted: {ACCEPTED_HASHES}", + file=sys.stderr, + ) sys.exit(1) db = load_database(args.db) if not args.platform and not args.emulator: @@ -4855,7 +4857,9 @@ from packverify import ( # noqa: E402,F401 # _OFFLINE is deliberately not re-exported: importing a value copies it, so a # facade name would keep saying False after set_offline() changed the flag. from packresolve import ( # noqa: E402,F401 + ACCEPTED_HASHES, _detect_hash_type, + sha1_for_hash, preferred_entries as _preferred_entries, parse_hash_input, parse_hash_file, diff --git a/scripts/packresolve.py b/scripts/packresolve.py index 33fe9e62..1d54413b 100644 --- a/scripts/packresolve.py +++ b/scripts/packresolve.py @@ -7,7 +7,6 @@ and the offline switch that forbids reaching for the network.""" from __future__ import annotations import re -_HEX_RE = re.compile(r"\b([0-9a-fA-F]{8,40})\b") from largefiles import fetch_large_file import os from common import ( @@ -81,43 +80,101 @@ def _verified_asset( return None -def _detect_hash_type(h: str) -> str: - n = len(h) - if n == 40: - return "sha1" - if n == 32: - return "md5" - if n == 8: - return "crc32" - return "md5" +# Digest length in hex digits -> kind. 29 is the MD5 Batocera writes with +# its last three digits cut, matched by prefix as the resolver does. +_HASH_KINDS = {8: "crc32", 29: "md5", 32: "md5", 40: "sha1", 64: "sha256"} +_HEX_DIGITS = re.compile(r"[0-9a-f]+") +_WORD = re.compile(r"[0-9A-Za-z]+") +ACCEPTED_HASHES = ( + "CRC32 (8 hex digits), MD5 (32, or the 29 Batocera writes), " + "SHA-1 (40) or SHA-256 (64)" +) + + +def _detect_hash_type(h: str) -> str | None: + """The digest a hex string is, by its length, or None.""" + return _HASH_KINDS.get(len(h)) + + +def _hash_token(token: str) -> tuple[str, str] | None: + """(kind, value) when a token is a whole digest of a known length.""" + value = token.strip().lower().removeprefix("*").removeprefix("0x") + kind = _detect_hash_type(value) + if kind and _HEX_DIGITS.fullmatch(value): + return kind, value + return None + + +def _refuse(rejected: list[str]) -> None: + shown = ", ".join(rejected[:5]) + (", ..." if len(rejected) > 5 else "") + raise ValueError(f"not a hash: {shown}. Accepted: {ACCEPTED_HASHES}") + def parse_hash_input(raw: str) -> list[tuple[str, str]]: - """Parse comma-separated hash string into (type, hash) tuples.""" + """Parse a comma-separated hash string into (type, hash) tuples. + + A part that is no digest is refused rather than dropped: a 12-digit + value was looked up as an MD5 and reported NOT FOUND, and a SHA-256 + vanished without a word. + """ results: list[tuple[str, str]] = [] + rejected: list[str] = [] for part in raw.split(","): - part = part.strip().lower() - if not part: + if not part.strip(): continue - m = _HEX_RE.search(part) - if m: - h = m.group(1) - results.append((_detect_hash_type(h), h)) + parsed = _hash_token(part) + if parsed: + results.append(parsed) + else: + rejected.append(part.strip()) + if rejected: + _refuse(rejected) return results + def parse_hash_file(path: str) -> list[tuple[str, str]]: - """Parse hash file (one per line, comments with #, mixed formats).""" + """Parse a hash file: one hash per line, alone or beside a file name as + md5sum writes it, ``#`` for comments. A line holding no digest is + refused with its line number.""" results: list[tuple[str, str]] = [] + rejected: list[str] = [] with open(path) as f: - for line in f: + for number, line in enumerate(f, 1): line = line.strip() if not line or line.startswith("#"): continue - m = _HEX_RE.search(line.lower()) - if m: - h = m.group(1) - results.append((_detect_hash_type(h), h)) + parsed = next( + (h for h in map(_hash_token, _WORD.findall(line)) if h), None + ) + if parsed: + results.append(parsed) + else: + rejected.append(f"line {number} ({line[:40]})") + if rejected: + _refuse(rejected) return results + +def sha1_for_hash(hash_type: str, hash_val: str, db: dict) -> str | None: + """The database key a looked-up hash names, or None.""" + files_db = db.get("files", {}) + indexes = db.get("indexes", {}) + if hash_type == "sha1": + sha1 = hash_val + elif hash_type == "crc32": + sha1 = indexes.get("by_crc32", {}).get(hash_val) + elif hash_type == "md5": + by_md5 = indexes.get("by_md5", {}) + sha1 = by_md5.get(hash_val) + if sha1 is None and len(hash_val) < 32: + prefixed = {v for k, v in by_md5.items() if k.startswith(hash_val)} + sha1 = prefixed.pop() if len(prefixed) == 1 else None + else: + sha1 = next( + (k for k, e in files_db.items() if e.get("sha256") == hash_val), None + ) + return sha1 if sha1 in files_db else None + def lookup_hashes( hashes: list[tuple[str, str]], db: dict, @@ -128,19 +185,10 @@ def lookup_hashes( """Print diagnostic info for each hash and return how many were not found.""" missing = 0 files_db = db.get("files", {}) - by_md5 = db.get("indexes", {}).get("by_md5", {}) - by_crc32 = db.get("indexes", {}).get("by_crc32", {}) for hash_type, hash_val in hashes: - sha1 = None - if hash_type == "sha1" and hash_val in files_db: - sha1 = hash_val - elif hash_type == "md5": - sha1 = by_md5.get(hash_val) - elif hash_type == "crc32": - sha1 = by_crc32.get(hash_val) - - if not sha1 or sha1 not in files_db: + sha1 = sha1_for_hash(hash_type, hash_val, db) + if not sha1: print(f"\n{hash_type.upper()}: {hash_val}") print(" NOT FOUND in database") missing += 1 diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 49c738e1..7b204060 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5906,5 +5906,44 @@ class PlatformSystemTypoGetsTheClosestNames(unittest.TestCase): self.assertNotIn("3do", printed) +class HashInputNamesWhatItAccepts(unittest.TestCase): + """A 12-digit value was looked up as an MD5 and reported NOT FOUND, a + SHA-256 vanished from the input, and the run said only "No valid hashes + found in input".""" + + def test_digests_are_read_by_length(self): + from scripts.packresolve import parse_hash_input + + sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + self.assertEqual(parse_hash_input(f"0x1A2B3C4D,{sha256}"), [ + ("crc32", "1a2b3c4d"), ("sha256", sha256), + ]) + self.assertEqual(parse_hash_input("a" * 29), [("md5", "a" * 29)]) + + def test_anything_else_is_refused_with_the_formats(self): + from scripts.packresolve import parse_hash_file, parse_hash_input + + for raw in ("1234567890ab", "zz", "490f666e1afb15b7362b406ed1cea246,zz"): + with self.subTest(raw=raw), self.assertRaises(ValueError) as ctx: + parse_hash_input(raw) + self.assertIn("SHA-256 (64)", str(ctx.exception)) + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as tmp: + listing = Path(tmp, "hashes.txt") + listing.write_text("# list\n490f666e1afb15b7362b406ed1cea246 a.bin\nb.bin\n") + with self.assertRaises(ValueError) as ctx: + parse_hash_file(str(listing)) + self.assertIn("line 3 (b.bin)", str(ctx.exception)) + + def test_a_sha256_finds_its_file(self): + from scripts.packresolve import sha1_for_hash + + db = json.loads((ROOT / "database.json").read_text(encoding="utf-8")) + sha1, entry = next( + (k, e) for k, e in db["files"].items() if e.get("sha256") + ) + self.assertEqual(sha1_for_hash("sha256", entry["sha256"], db), sha1) + self.assertEqual(sha1_for_hash("md5", entry["md5"][:29], db), sha1) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/tools.md b/wiki/tools.md index 709cfa06..3db2d9df 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -217,10 +217,14 @@ would have loaded. `required: false`, keep only required - `--split`: generate one ZIP per system instead of one big pack - `--split --group-by manufacturer`: group split packs by manufacturer (Sony, Nintendo, Sega...) -- `--from-md5`: look up a hash in the database, or build a custom pack with `--platform`/`--emulator` -- `--from-md5-file`: same, reading hashes from a file: one MD5, SHA-1 or CRC32 - per line, alone or beside a file name as `md5sum` writes it, `#` for comments. - A lookup exits 1 when a hash is not in the database +- `--from-md5`: look up a hash in the database, or build a custom pack with + `--platform`/`--emulator`. Any digest is read by its length: CRC32 (8 hex + digits), MD5 (32, or the 29 Batocera writes), SHA-1 (40), SHA-256 (64). + Anything else is refused with the value that is not a hash +- `--from-md5-file`: same, reading hashes from a file: one per line, alone or + beside a file name as `md5sum` writes it, `#` for comments. A line with no + hash is refused with its number. A lookup exits 1 when a hash is not in the + database - `--target`: filter by hardware target (e.g. `switch`, `rpi4`, `x86_64`) - `--region`: ordered priority list, best first. Regions nest, a system with no regional split keeps everything, and a group with no candidate in any named