From ae3dc63bc34ba5778b08011fcf97c27d0b24a912 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Sun, 18 Oct 2026 07:24:33 +0200 Subject: [PATCH] fix: accept RomM files on size and any hash RomM's verify_file_hashes compares the size, then md5, sha1 or crc. We compared md5 alone, so a fixture entry with one mistyped hash was left out of the pack RomM would accept it from. The rule is declared in romm.yml and read by verify, the builder and the manifest. --- platforms/romm.yml | 5 +- schemas/platform.schema.json | 7 +++ scripts/generate_pack.py | 9 ++-- scripts/nativemode.py | 6 +++ scripts/packverify.py | 6 ++- scripts/scraper/romm_scraper.py | 2 + scripts/validation.py | 39 +++++++++++--- scripts/verify.py | 25 +++++++-- tests/test_romm_hash_rule.py | 90 +++++++++++++++++++++++++++++++++ wiki/verification-modes.md | 13 +++-- 10 files changed, 182 insertions(+), 20 deletions(-) create mode 100644 tests/test_romm_hash_rule.py diff --git a/platforms/romm.yml b/platforms/romm.yml index bc3c7eba..ab7db6cb 100644 --- a/platforms/romm.yml +++ b/platforms/romm.yml @@ -1,11 +1,12 @@ inherits: emulatorjs platform: RomM -version: 5.3.1 +version: 5.4.0 homepage: "https://romm.app" -source: "https://raw.githubusercontent.com/rommapp/romm/5.3.1/backend/models/fixtures/known_bios_files.json" +source: "https://raw.githubusercontent.com/rommapp/romm/5.4.0/backend/models/fixtures/known_bios_files.json" base_destination: bios hash_type: sha1 verification_mode: md5 +hash_rule: size_and_any_hash systems: 3do: files: diff --git a/schemas/platform.schema.json b/schemas/platform.schema.json index 1523af8a..30518648 100644 --- a/schemas/platform.schema.json +++ b/schemas/platform.schema.json @@ -68,6 +68,13 @@ ], "description": "Primary hash algorithm used by this platform" }, + "hash_rule": { + "type": "string", + "enum": [ + "size_and_any_hash" + ], + "description": "How a digest-mode frontend accepts a file when it is not one digest: size_and_any_hash requires the declared size, then any one of md5, sha1 and crc32 (RomM)" + }, "verification_mode": { "type": "string", "enum": [ diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 755482c6..d69656fc 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -805,7 +805,8 @@ def generate_pack( # one digest accepts a file another declared hash # contradicts, as verify judges it. conflict_status, refusal = baseline_verdict( - file_entry, conflict_path, conflict_status, verification_mode + file_entry, conflict_path, conflict_status, verification_mode, + config.get("hash_rule", ""), ) if conflict_status == "not_found": file_status[dedup_key] = "missing" @@ -937,7 +938,8 @@ def generate_pack( # The verdict the install manifest reads too: a member missing # from a zipped_file archive, or a hash the frontend rejects. status, refusal = baseline_verdict( - file_entry, local_path, status, verification_mode + file_entry, local_path, status, verification_mode, + config.get("hash_rule", ""), ) if refusal: file_status[dedup_key] = "excluded" @@ -3949,7 +3951,8 @@ def generate_manifest( # missing whatever the outer resolution said. An existence # platform never reads the bytes. status, refusal = baseline_verdict( - file_entry, local_path, status, verification_mode + file_entry, local_path, status, verification_mode, + config.get("hash_rule", ""), ) if refusal or status in ("not_found", "external"): return None, "hash_mismatch" if refusal else status diff --git a/scripts/nativemode.py b/scripts/nativemode.py index 0e90d467..523f3358 100644 --- a/scripts/nativemode.py +++ b/scripts/nativemode.py @@ -34,6 +34,12 @@ MODES = ("existence", "md5", "sha1") MODE_DIGEST = {"md5": "md5", "sha1": "sha1"} +#: A frontend that accepts a file on its size and any one declared hash. +#: RomM's Firmware.verify_file_hashes: size first, then md5, sha1 or crc. +SIZE_AND_ANY_HASH = "size_and_any_hash" +HASH_RULES = (SIZE_AND_ANY_HASH,) + + def normalize(mode: str | None) -> str: """Return a declared mode, falling back to the schema default.""" return mode if mode in MODES else DEFAULT_MODE diff --git a/scripts/packverify.py b/scripts/packverify.py index 7cf8a1cf..fffa361f 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -464,6 +464,7 @@ def _intentional_hash_exclusion( zip_contents: dict | None = None, data_dir_registry: dict | None = None, verification_mode: str = "md5", + hash_rule: str = "", ) -> bool: """Return whether the builder must omit every declaration as unsafe. @@ -490,7 +491,9 @@ def _intentional_hash_exclusion( # The builder's own verdict: a zipped_file archive is refused when the # member is missing whatever the outer resolution said, a hash # mismatch when the frontend's digest disagrees too. - _status, refusal = baseline_verdict(entry, local_path, status, verification_mode) + _status, refusal = baseline_verdict( + entry, local_path, status, verification_mode, hash_rule + ) if refusal is None: return False return True @@ -652,6 +655,7 @@ def verify_pack_against_platform( zip_contents=exclusion_index, data_dir_registry=data_registry, verification_mode=verification_mode, + hash_rule=config.get("hash_rule", ""), ): baseline_excluded += 1 continue diff --git a/scripts/scraper/romm_scraper.py b/scripts/scraper/romm_scraper.py index a7608220..38bed386 100644 --- a/scripts/scraper/romm_scraper.py +++ b/scripts/scraper/romm_scraper.py @@ -221,6 +221,8 @@ class Scraper(BaseScraper): "base_destination": "bios", "hash_type": "sha1", "verification_mode": "md5", + # Firmware.verify_file_hashes: size, then md5, sha1 or crc. + "hash_rule": "size_and_any_hash", "systems": systems, } diff --git a/scripts/validation.py b/scripts/validation.py index e80eeba3..f4fe917f 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -19,7 +19,7 @@ from common import ( size_fits, ) from hashing import md5_composite, parse_md5_list -from nativemode import digest_algorithm, hash_mismatch_excludes_file +from nativemode import SIZE_AND_ANY_HASH, digest_algorithm, hash_mismatch_excludes_file from ziptools import check_inside_zip # Validation types that require console-specific cryptographic keys. @@ -649,15 +649,30 @@ def agnostic_substitute( return None -def frontend_digest_matches(file_entry: dict, local_path: str, algorithm: str) -> bool: +def frontend_digest_matches( + file_entry: dict, local_path: str, algorithm: str, rule: str = "" +) -> bool: """Whether the frontend's own digest accepts the file. resolve_local_file calls a file `hash_mismatch` when any declared hash - disagrees, but a frontend compares one digest. RomM's list carries md5, - sha1 and crc32: a mistyped crc32 next to the right md5 must not withhold - a file RomM accepts. verify_entry_md5 reads it too: Batocera compares a - truncated md5 by prefix and Recalbox hashes a ZIP by its members. + disagrees, but a frontend compares one digest. verify_entry_md5 reads it + too: Batocera compares a truncated md5 by prefix and Recalbox hashes a + ZIP by its members. Under SIZE_AND_ANY_HASH (RomM) the declared size must + match and then any one of md5, sha1 and crc32 does: a mistyped hash next + to a right one does not withhold the file, a wrong size does. """ + if rule == SIZE_AND_ANY_HASH: + if not local_path: + return False + # RomM reads an absent size as 0: such an entry never verifies. + if os.path.getsize(local_path) != int(file_entry.get("size") or 0): + return False + actual = compute_hashes(local_path) + return any( + actual[field].lower() == str(file_entry[field]).strip().lower() + for field in ("md5", "sha1", "crc32") + if file_entry.get(field) + ) if algorithm == "md5": declared = parse_md5_list(file_entry.get("md5")) else: @@ -698,7 +713,11 @@ def inner_rom_check(file_entry: dict, local_path: str) -> str: def baseline_verdict( - file_entry: dict, local_path: str | None, status: str, verification_mode: str + file_entry: dict, + local_path: str | None, + status: str, + verification_mode: str, + rule: str = "", ) -> tuple[str, str | None]: """The status a platform entry ships under, or why a hash platform refuses it. @@ -721,6 +740,12 @@ def baseline_verdict( if result == "error": return status, "cannot read ZIP" return status, f"{member} MD5 mismatch inside ZIP" + if rule == SIZE_AND_ANY_HASH and local_path: + # A hash found by the resolver does not settle it: the size the + # frontend compares first can still be declared otherwise. + if frontend_digest_matches(file_entry, local_path, "", rule): + return status, None + return status, "size or hash mismatch" if status != "hash_mismatch": return status, None if frontend_digest_matches(file_entry, local_path, digest_algorithm(verification_mode)): diff --git a/scripts/verify.py b/scripts/verify.py index 0a86fe1a..0211d8dc 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -70,6 +70,7 @@ from common import ( yaml = require_yaml() from nativemode import ( + SIZE_AND_ANY_HASH, digest_algorithm, hash_mismatch_excludes_file, normalize as normalize_mode, @@ -173,8 +174,12 @@ def verify_entry_md5( file_entry: dict, local_path: str | None, resolve_status: str = "", + hash_rule: str = "", ) -> dict: - """MD5 verification -Batocera md5sum + Recalbox multi-hash + Md5Composite.""" + """MD5 verification -Batocera md5sum + Recalbox multi-hash + Md5Composite. + + Under SIZE_AND_ANY_HASH (RomM) the size and any one declared hash decide. + """ name = file_entry.get("name", "") expected_md5 = file_entry.get("md5", "") zipped_file = file_entry.get("zipped_file") @@ -189,6 +194,16 @@ def verify_entry_md5( if not local_path: return {**base, "status": Status.MISSING} + if hash_rule == SIZE_AND_ANY_HASH: + if frontend_digest_matches(file_entry, local_path, "md5", hash_rule): + return {**base, "status": Status.OK, "path": local_path} + return { + **base, + "status": Status.UNTESTED, + "path": local_path, + "reason": "the declared size and at least one declared hash must match", + } + if zipped_file: found_in_zip = False had_error = False @@ -1034,7 +1049,9 @@ def verify_platform( elif digest_algorithm(mode) == "sha1": result = verify_entry_sha1(file_entry, local_path) else: - result = verify_entry_md5(file_entry, local_path, resolve_status) + result = verify_entry_md5( + file_entry, local_path, resolve_status, config.get("hash_rule", "") + ) # Emulator-level validation: informational for platform packs. # Platform verification (MD5) is the authority. Emulator # mismatches are reported as discrepancies, not failures. @@ -1058,7 +1075,9 @@ def verify_platform( if ( local_path and result["status"] != Status.MISSING - and baseline_verdict(file_entry, local_path, resolve_status, mode)[1] is None + and baseline_verdict( + file_entry, local_path, resolve_status, mode, config.get("hash_rule", "") + )[1] is None ): shipped.setdefault( f"{base_dest}/{destination}" if base_dest else destination, diff --git a/tests/test_romm_hash_rule.py b/tests/test_romm_hash_rule.py new file mode 100644 index 00000000..922d83f9 --- /dev/null +++ b/tests/test_romm_hash_rule.py @@ -0,0 +1,90 @@ +"""RomM accepts a firmware file on its size and any one declared hash. + +Firmware.verify_file_hashes compares file_size_bytes first, then md5, sha1 +or crc. verify and the builder compared the md5 alone: a fixture entry with a +mistyped md5 beside the right sha1 was refused here and accepted by RomM, and +a wrong declared size passed here and failed there. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import tempfile +import unittest +import zlib +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from nativemode import HASH_RULES, SIZE_AND_ANY_HASH # noqa: E402 +from validation import baseline_verdict, frontend_digest_matches # noqa: E402 +from verify import Status, verify_entry_md5 # noqa: E402 + +PAYLOAD = b"romm firmware" + + +class SizeAndAnyHash(unittest.TestCase): + def setUp(self): + scratch = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") + self.addCleanup(scratch.cleanup) + self.path = str(Path(scratch.name) / "fw.bin") + Path(self.path).write_bytes(PAYLOAD) + self.entry = { + "name": "fw.bin", + "size": len(PAYLOAD), + "md5": hashlib.md5(PAYLOAD).hexdigest(), + "sha1": hashlib.sha1(PAYLOAD).hexdigest(), + "crc32": f"{zlib.crc32(PAYLOAD):08x}", + } + + def _accepts(self, **changes) -> bool: + return frontend_digest_matches( + {**self.entry, **changes}, self.path, "md5", SIZE_AND_ANY_HASH + ) + + def test_a_mistyped_md5_beside_the_right_sha1_is_accepted(self): + self.assertTrue(self._accepts(md5="0" * 32)) + self.assertFalse(frontend_digest_matches({**self.entry, "md5": "0" * 32}, self.path, "md5")) + + def test_a_wrong_declared_size_is_refused_whatever_the_hashes(self): + self.assertFalse(self._accepts(size=len(PAYLOAD) + 1)) + + def test_an_entry_without_a_size_never_verifies(self): + entry = dict(self.entry) + del entry["size"] + self.assertFalse(frontend_digest_matches(entry, self.path, "md5", SIZE_AND_ANY_HASH)) + + def test_no_hash_matching_is_refused(self): + self.assertFalse(self._accepts(md5="0" * 32, sha1="0" * 40, crc32="00000000")) + + def test_the_builder_refuses_a_hash_exact_file_at_the_wrong_size(self): + entry = {**self.entry, "size": 1} + _status, refusal = baseline_verdict(entry, self.path, "md5_exact", "md5", SIZE_AND_ANY_HASH) + self.assertIsNotNone(refusal) + _status, refusal = baseline_verdict(entry, self.path, "md5_exact", "md5") + self.assertIsNone(refusal) + + def test_verify_reads_the_same_rule(self): + accepted = verify_entry_md5({**self.entry, "md5": "0" * 32}, self.path, "hash_mismatch", + SIZE_AND_ANY_HASH) + self.assertEqual(accepted["status"], Status.OK) + refused = verify_entry_md5({**self.entry, "size": 1}, self.path, "md5_exact", + SIZE_AND_ANY_HASH) + self.assertEqual(refused["status"], Status.UNTESTED) + + +class TheRuleIsDeclared(unittest.TestCase): + def test_romm_declares_it_and_the_schema_knows_it(self): + import yaml + + romm = yaml.safe_load((REPO_ROOT / "platforms" / "romm.yml").read_text(encoding="utf-8")) + self.assertEqual(romm.get("hash_rule"), SIZE_AND_ANY_HASH) + schema = json.loads((REPO_ROOT / "schemas" / "platform.schema.json").read_text(encoding="utf-8")) + self.assertEqual(tuple(schema["properties"]["hash_rule"]["enum"]), HASH_RULES) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index 3413960d..3007af67 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -93,10 +93,15 @@ If the inner file is not found inside the ZIP, the status is UNTESTED with a rea ### RomM verification -RomM uses MD5 verification (`verification_mode: md5`). The platform YAML stores -SHA1, MD5, and CRC32 for reference, but `verify.py` checks only the MD5 field, -matching the platform's runtime behavior. ZIP files are not opened; only the -container is checked. +**Source**: `backend/models/firmware.py`, `Firmware.verify_file_hashes` + +RomM reads the file's bytes (`verification_mode: md5`) but does not compare one +digest. Its platform YAML carries `hash_rule: size_and_any_hash`: the file's +size must equal the declared size, then any one of MD5, SHA1 and CRC32 must +match. An entry without a size never verifies, since RomM reads it as 0. A +wrong hash beside a right one does not withhold the file. `verify.py`, the +pack builder and the install manifest apply that rule. ZIP files are not +opened; only the container is checked. ### ROCKNIX verification