diff --git a/platforms/romm.yml b/platforms/romm.yml index bc3c7eba..ab7db6cb 100644 --- a/platforms/romm.yml +++ b/platforms/romm.yml @@ -1,11 +1,12 @@ inherits: emulatorjs platform: RomM -version: 5.3.1 +version: 5.4.0 homepage: "https://romm.app" -source: "https://raw.githubusercontent.com/rommapp/romm/5.3.1/backend/models/fixtures/known_bios_files.json" +source: "https://raw.githubusercontent.com/rommapp/romm/5.4.0/backend/models/fixtures/known_bios_files.json" base_destination: bios hash_type: sha1 verification_mode: md5 +hash_rule: size_and_any_hash systems: 3do: files: diff --git a/schemas/platform.schema.json b/schemas/platform.schema.json index 1523af8a..30518648 100644 --- a/schemas/platform.schema.json +++ b/schemas/platform.schema.json @@ -68,6 +68,13 @@ ], "description": "Primary hash algorithm used by this platform" }, + "hash_rule": { + "type": "string", + "enum": [ + "size_and_any_hash" + ], + "description": "How a digest-mode frontend accepts a file when it is not one digest: size_and_any_hash requires the declared size, then any one of md5, sha1 and crc32 (RomM)" + }, "verification_mode": { "type": "string", "enum": [ diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 755482c6..d69656fc 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -805,7 +805,8 @@ def generate_pack( # one digest accepts a file another declared hash # contradicts, as verify judges it. conflict_status, refusal = baseline_verdict( - file_entry, conflict_path, conflict_status, verification_mode + file_entry, conflict_path, conflict_status, verification_mode, + config.get("hash_rule", ""), ) if conflict_status == "not_found": file_status[dedup_key] = "missing" @@ -937,7 +938,8 @@ def generate_pack( # The verdict the install manifest reads too: a member missing # from a zipped_file archive, or a hash the frontend rejects. status, refusal = baseline_verdict( - file_entry, local_path, status, verification_mode + file_entry, local_path, status, verification_mode, + config.get("hash_rule", ""), ) if refusal: file_status[dedup_key] = "excluded" @@ -3949,7 +3951,8 @@ def generate_manifest( # missing whatever the outer resolution said. An existence # platform never reads the bytes. status, refusal = baseline_verdict( - file_entry, local_path, status, verification_mode + file_entry, local_path, status, verification_mode, + config.get("hash_rule", ""), ) if refusal or status in ("not_found", "external"): return None, "hash_mismatch" if refusal else status diff --git a/scripts/nativemode.py b/scripts/nativemode.py index 0e90d467..523f3358 100644 --- a/scripts/nativemode.py +++ b/scripts/nativemode.py @@ -34,6 +34,12 @@ MODES = ("existence", "md5", "sha1") MODE_DIGEST = {"md5": "md5", "sha1": "sha1"} +#: A frontend that accepts a file on its size and any one declared hash. +#: RomM's Firmware.verify_file_hashes: size first, then md5, sha1 or crc. +SIZE_AND_ANY_HASH = "size_and_any_hash" +HASH_RULES = (SIZE_AND_ANY_HASH,) + + def normalize(mode: str | None) -> str: """Return a declared mode, falling back to the schema default.""" return mode if mode in MODES else DEFAULT_MODE diff --git a/scripts/packverify.py b/scripts/packverify.py index 7cf8a1cf..fffa361f 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -464,6 +464,7 @@ def _intentional_hash_exclusion( zip_contents: dict | None = None, data_dir_registry: dict | None = None, verification_mode: str = "md5", + hash_rule: str = "", ) -> bool: """Return whether the builder must omit every declaration as unsafe. @@ -490,7 +491,9 @@ def _intentional_hash_exclusion( # The builder's own verdict: a zipped_file archive is refused when the # member is missing whatever the outer resolution said, a hash # mismatch when the frontend's digest disagrees too. - _status, refusal = baseline_verdict(entry, local_path, status, verification_mode) + _status, refusal = baseline_verdict( + entry, local_path, status, verification_mode, hash_rule + ) if refusal is None: return False return True @@ -652,6 +655,7 @@ def verify_pack_against_platform( zip_contents=exclusion_index, data_dir_registry=data_registry, verification_mode=verification_mode, + hash_rule=config.get("hash_rule", ""), ): baseline_excluded += 1 continue diff --git a/scripts/scraper/romm_scraper.py b/scripts/scraper/romm_scraper.py index a7608220..38bed386 100644 --- a/scripts/scraper/romm_scraper.py +++ b/scripts/scraper/romm_scraper.py @@ -221,6 +221,8 @@ class Scraper(BaseScraper): "base_destination": "bios", "hash_type": "sha1", "verification_mode": "md5", + # Firmware.verify_file_hashes: size, then md5, sha1 or crc. + "hash_rule": "size_and_any_hash", "systems": systems, } diff --git a/scripts/validation.py b/scripts/validation.py index e80eeba3..f4fe917f 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -19,7 +19,7 @@ from common import ( size_fits, ) from hashing import md5_composite, parse_md5_list -from nativemode import digest_algorithm, hash_mismatch_excludes_file +from nativemode import SIZE_AND_ANY_HASH, digest_algorithm, hash_mismatch_excludes_file from ziptools import check_inside_zip # Validation types that require console-specific cryptographic keys. @@ -649,15 +649,30 @@ def agnostic_substitute( return None -def frontend_digest_matches(file_entry: dict, local_path: str, algorithm: str) -> bool: +def frontend_digest_matches( + file_entry: dict, local_path: str, algorithm: str, rule: str = "" +) -> bool: """Whether the frontend's own digest accepts the file. resolve_local_file calls a file `hash_mismatch` when any declared hash - disagrees, but a frontend compares one digest. RomM's list carries md5, - sha1 and crc32: a mistyped crc32 next to the right md5 must not withhold - a file RomM accepts. verify_entry_md5 reads it too: Batocera compares a - truncated md5 by prefix and Recalbox hashes a ZIP by its members. + disagrees, but a frontend compares one digest. verify_entry_md5 reads it + too: Batocera compares a truncated md5 by prefix and Recalbox hashes a + ZIP by its members. Under SIZE_AND_ANY_HASH (RomM) the declared size must + match and then any one of md5, sha1 and crc32 does: a mistyped hash next + to a right one does not withhold the file, a wrong size does. """ + if rule == SIZE_AND_ANY_HASH: + if not local_path: + return False + # RomM reads an absent size as 0: such an entry never verifies. + if os.path.getsize(local_path) != int(file_entry.get("size") or 0): + return False + actual = compute_hashes(local_path) + return any( + actual[field].lower() == str(file_entry[field]).strip().lower() + for field in ("md5", "sha1", "crc32") + if file_entry.get(field) + ) if algorithm == "md5": declared = parse_md5_list(file_entry.get("md5")) else: @@ -698,7 +713,11 @@ def inner_rom_check(file_entry: dict, local_path: str) -> str: def baseline_verdict( - file_entry: dict, local_path: str | None, status: str, verification_mode: str + file_entry: dict, + local_path: str | None, + status: str, + verification_mode: str, + rule: str = "", ) -> tuple[str, str | None]: """The status a platform entry ships under, or why a hash platform refuses it. @@ -721,6 +740,12 @@ def baseline_verdict( if result == "error": return status, "cannot read ZIP" return status, f"{member} MD5 mismatch inside ZIP" + if rule == SIZE_AND_ANY_HASH and local_path: + # A hash found by the resolver does not settle it: the size the + # frontend compares first can still be declared otherwise. + if frontend_digest_matches(file_entry, local_path, "", rule): + return status, None + return status, "size or hash mismatch" if status != "hash_mismatch": return status, None if frontend_digest_matches(file_entry, local_path, digest_algorithm(verification_mode)): diff --git a/scripts/verify.py b/scripts/verify.py index 0a86fe1a..0211d8dc 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -70,6 +70,7 @@ from common import ( yaml = require_yaml() from nativemode import ( + SIZE_AND_ANY_HASH, digest_algorithm, hash_mismatch_excludes_file, normalize as normalize_mode, @@ -173,8 +174,12 @@ def verify_entry_md5( file_entry: dict, local_path: str | None, resolve_status: str = "", + hash_rule: str = "", ) -> dict: - """MD5 verification -Batocera md5sum + Recalbox multi-hash + Md5Composite.""" + """MD5 verification -Batocera md5sum + Recalbox multi-hash + Md5Composite. + + Under SIZE_AND_ANY_HASH (RomM) the size and any one declared hash decide. + """ name = file_entry.get("name", "") expected_md5 = file_entry.get("md5", "") zipped_file = file_entry.get("zipped_file") @@ -189,6 +194,16 @@ def verify_entry_md5( if not local_path: return {**base, "status": Status.MISSING} + if hash_rule == SIZE_AND_ANY_HASH: + if frontend_digest_matches(file_entry, local_path, "md5", hash_rule): + return {**base, "status": Status.OK, "path": local_path} + return { + **base, + "status": Status.UNTESTED, + "path": local_path, + "reason": "the declared size and at least one declared hash must match", + } + if zipped_file: found_in_zip = False had_error = False @@ -1034,7 +1049,9 @@ def verify_platform( elif digest_algorithm(mode) == "sha1": result = verify_entry_sha1(file_entry, local_path) else: - result = verify_entry_md5(file_entry, local_path, resolve_status) + result = verify_entry_md5( + file_entry, local_path, resolve_status, config.get("hash_rule", "") + ) # Emulator-level validation: informational for platform packs. # Platform verification (MD5) is the authority. Emulator # mismatches are reported as discrepancies, not failures. @@ -1058,7 +1075,9 @@ def verify_platform( if ( local_path and result["status"] != Status.MISSING - and baseline_verdict(file_entry, local_path, resolve_status, mode)[1] is None + and baseline_verdict( + file_entry, local_path, resolve_status, mode, config.get("hash_rule", "") + )[1] is None ): shipped.setdefault( f"{base_dest}/{destination}" if base_dest else destination, diff --git a/tests/test_romm_hash_rule.py b/tests/test_romm_hash_rule.py new file mode 100644 index 00000000..922d83f9 --- /dev/null +++ b/tests/test_romm_hash_rule.py @@ -0,0 +1,90 @@ +"""RomM accepts a firmware file on its size and any one declared hash. + +Firmware.verify_file_hashes compares file_size_bytes first, then md5, sha1 +or crc. verify and the builder compared the md5 alone: a fixture entry with a +mistyped md5 beside the right sha1 was refused here and accepted by RomM, and +a wrong declared size passed here and failed there. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +import tempfile +import unittest +import zlib +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from nativemode import HASH_RULES, SIZE_AND_ANY_HASH # noqa: E402 +from validation import baseline_verdict, frontend_digest_matches # noqa: E402 +from verify import Status, verify_entry_md5 # noqa: E402 + +PAYLOAD = b"romm firmware" + + +class SizeAndAnyHash(unittest.TestCase): + def setUp(self): + scratch = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") + self.addCleanup(scratch.cleanup) + self.path = str(Path(scratch.name) / "fw.bin") + Path(self.path).write_bytes(PAYLOAD) + self.entry = { + "name": "fw.bin", + "size": len(PAYLOAD), + "md5": hashlib.md5(PAYLOAD).hexdigest(), + "sha1": hashlib.sha1(PAYLOAD).hexdigest(), + "crc32": f"{zlib.crc32(PAYLOAD):08x}", + } + + def _accepts(self, **changes) -> bool: + return frontend_digest_matches( + {**self.entry, **changes}, self.path, "md5", SIZE_AND_ANY_HASH + ) + + def test_a_mistyped_md5_beside_the_right_sha1_is_accepted(self): + self.assertTrue(self._accepts(md5="0" * 32)) + self.assertFalse(frontend_digest_matches({**self.entry, "md5": "0" * 32}, self.path, "md5")) + + def test_a_wrong_declared_size_is_refused_whatever_the_hashes(self): + self.assertFalse(self._accepts(size=len(PAYLOAD) + 1)) + + def test_an_entry_without_a_size_never_verifies(self): + entry = dict(self.entry) + del entry["size"] + self.assertFalse(frontend_digest_matches(entry, self.path, "md5", SIZE_AND_ANY_HASH)) + + def test_no_hash_matching_is_refused(self): + self.assertFalse(self._accepts(md5="0" * 32, sha1="0" * 40, crc32="00000000")) + + def test_the_builder_refuses_a_hash_exact_file_at_the_wrong_size(self): + entry = {**self.entry, "size": 1} + _status, refusal = baseline_verdict(entry, self.path, "md5_exact", "md5", SIZE_AND_ANY_HASH) + self.assertIsNotNone(refusal) + _status, refusal = baseline_verdict(entry, self.path, "md5_exact", "md5") + self.assertIsNone(refusal) + + def test_verify_reads_the_same_rule(self): + accepted = verify_entry_md5({**self.entry, "md5": "0" * 32}, self.path, "hash_mismatch", + SIZE_AND_ANY_HASH) + self.assertEqual(accepted["status"], Status.OK) + refused = verify_entry_md5({**self.entry, "size": 1}, self.path, "md5_exact", + SIZE_AND_ANY_HASH) + self.assertEqual(refused["status"], Status.UNTESTED) + + +class TheRuleIsDeclared(unittest.TestCase): + def test_romm_declares_it_and_the_schema_knows_it(self): + import yaml + + romm = yaml.safe_load((REPO_ROOT / "platforms" / "romm.yml").read_text(encoding="utf-8")) + self.assertEqual(romm.get("hash_rule"), SIZE_AND_ANY_HASH) + schema = json.loads((REPO_ROOT / "schemas" / "platform.schema.json").read_text(encoding="utf-8")) + self.assertEqual(tuple(schema["properties"]["hash_rule"]["enum"]), HASH_RULES) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index 3413960d..3007af67 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -93,10 +93,15 @@ If the inner file is not found inside the ZIP, the status is UNTESTED with a rea ### RomM verification -RomM uses MD5 verification (`verification_mode: md5`). The platform YAML stores -SHA1, MD5, and CRC32 for reference, but `verify.py` checks only the MD5 field, -matching the platform's runtime behavior. ZIP files are not opened; only the -container is checked. +**Source**: `backend/models/firmware.py`, `Firmware.verify_file_hashes` + +RomM reads the file's bytes (`verification_mode: md5`) but does not compare one +digest. Its platform YAML carries `hash_rule: size_and_any_hash`: the file's +size must equal the declared size, then any one of MD5, SHA1 and CRC32 must +match. An entry without a size never verifies, since RomM reads it as 0. A +wrong hash beside a right one does not withhold the file. `verify.py`, the +pack builder and the install manifest apply that rule. ZIP files are not +opened; only the container is checked. ### ROCKNIX verification