From d8e9ad23d33f53bf8fcb75ab87c88a27e2d77f6c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:54:57 +0200 Subject: [PATCH] docs: describe short md5 handling as the code does No platform YAML holds an md5 shorter than 32 characters: the Batocera scraper completes one when a single collection md5 starts with it. The pages said some entries carry 29 characters and that scrapers store them as-is. --- scripts/packverify.py | 4 ++-- tests/test_doc_pages.py | 26 ++++++++++++++++++++++++++ tests/test_e2e.py | 4 ++-- tests/test_resolution_evidence.py | 2 +- tests/test_shipped_content.py | 2 +- wiki/adding-a-scraper.md | 6 ++++-- wiki/architecture.md | 2 +- wiki/verification-modes.md | 13 +++++++------ 8 files changed, 44 insertions(+), 15 deletions(-) diff --git a/scripts/packverify.py b/scripts/packverify.py index 94c13046..c3628daf 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -314,8 +314,8 @@ def _hash_matches(declared: str, actual: str) -> bool: """Compare a declared hash value against an actual hex digest. Handles comma-separated multi-hash lists and uppercase (Recalbox) - and truncated MD5s (Batocera 29-char): a declared value shorter - than the digest matches by prefix. + and MD5s shorter than 32 characters: a declared value shorter than + the digest matches by prefix. """ actual = actual.lower() for cand in declared.split(","): diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 30903f5c..99669ff0 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -497,5 +497,31 @@ class EmulatorVerboseSample(unittest.TestCase): self.assertIn(line, printed) +class ShortMd5Handling(unittest.TestCase): + """Three pages said some Batocera entries carry 29-character MD5s and show + as untested. No platform YAML holds one, since the Batocera scraper + completes them, and adding-a-scraper.md said to store them as-is.""" + + def test_no_page_claims_short_md5_entries(self): + offenders = [ + page.name for page in PUBLISHED + if re.search(r"(?i)29[- ]char|truncated md5 hashes in some entries|store the hash as-is", + page.read_text(encoding="utf-8")) + ] + self.assertEqual(offenders, []) + + def test_the_page_names_what_the_code_does(self): + import common + from scraper import batocera_scraper + + scraper = inspect.getsource(batocera_scraper) + self.assertIn("md5 = _resolve_truncated_md5(", scraper) + resolver = inspect.getsource(common.resolve_local_file) + self.assertIn("if len(md5_candidate) < 32:", resolver) + self.assertIn("_md5_name_ok(path)", resolver) + section = _section(_page("verification-modes.md"), "MD5 shorter than 32 characters (Batocera)") + self.assertIn("_resolve_truncated_md5", section) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_e2e.py b/tests/test_e2e.py index f0d158bb..900dcabd 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -265,7 +265,7 @@ class TestE2E(unittest.TestCase): ).hexdigest() # sorted: a.rom, b.rom multi_wrong = "0000000000000000000000000000000" multi_right = f["multi.zip"]["inner_md5s"]["rom.bin"] - truncated_md5 = f["truncated.bin"]["md5"][:29] # Batocera 29-char + truncated_md5 = f["truncated.bin"]["md5"][:29] # shorter than 32 characters config = { "platform": "TestMD5", @@ -347,7 +347,7 @@ class TestE2E(unittest.TestCase): "zipped_file": "rom.bin", "required": True, }, - # Truncated MD5 (Batocera 29 chars) + # MD5 shorter than 32 characters { "name": "truncated.bin", "destination": "truncated.bin", diff --git a/tests/test_resolution_evidence.py b/tests/test_resolution_evidence.py index 4677c0bf..588a18be 100644 --- a/tests/test_resolution_evidence.py +++ b/tests/test_resolution_evidence.py @@ -86,7 +86,7 @@ class DeclaredHashVerdict(unittest.TestCase): ) def test_a_truncated_md5_still_decides(self): - """Batocera publishes 29-character prefixes, which are still evidence.""" + """An MD5 shorter than 32 characters is still evidence, by prefix.""" self.assertEqual( self.verdict(self.right, md5_list=[self.d["md5"][:29]]), "data_dir_hash_exact", diff --git a/tests/test_shipped_content.py b/tests/test_shipped_content.py index a1229ca9..75597b30 100644 --- a/tests/test_shipped_content.py +++ b/tests/test_shipped_content.py @@ -118,7 +118,7 @@ class ShippedFilesSatisfyTheirDeclaration(unittest.TestCase): digests["md5"].append(common.md5_composite(str(blob_path))) except (zipfile.BadZipFile, OSError): pass - # Batocera publishes 29-character prefixes, so compare by prefix. + # A declared MD5 can be shorter than 32 characters: compare by prefix. satisfied = any( all( any( diff --git a/wiki/adding-a-scraper.md b/wiki/adding-a-scraper.md index 584b1b8b..3010ef3e 100644 --- a/wiki/adding-a-scraper.md +++ b/wiki/adding-a-scraper.md @@ -206,8 +206,10 @@ System ID consistency matters for cross-platform operations. The same console mu - Handle missing hashes gracefully (set to `None`, not empty string). - Some platforms provide multiple hash types per entry. Populate whichever fields are available. -- Batocera uses 29-character truncated MD5 hashes in some entries. The resolution - layer handles prefix matching, but the scraper should store the hash as-is. +- An MD5 shorter than 32 characters is completed when exactly one MD5 in + `database.json` starts with it, as the Batocera scraper does + (`_resolve_truncated_md5`), and kept as written otherwise. The resolution layer + matches what remains by prefix. Scraped data reflects what the upstream declares, which may not match reality. The scraper's job is faithful transcription of upstream data, not correction. Corrections happen in the emulator profiles (source-verified) and in `_shared.yml` (curated). If a scraper meets a hash or filename that differs from the emulator source, log a warning and still include the upstream value. The divergence will surface during truth diffing. diff --git a/wiki/architecture.md b/wiki/architecture.md index 08ee1071..ffc09f34 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -372,7 +372,7 @@ platform's own verification mode. | Case | Handling | |------|---------| -| Batocera truncated MD5 (29 chars) | prefix match in resolution | +| MD5 shorter than 32 characters | completed by the Batocera scraper when one MD5 in the collection starts with it, matched by prefix in resolution | | `zippedFile` entries | MD5 is of the ROM inside the ZIP, not the ZIP itself | | Regional variants (same filename) | `by_path_suffix` index disambiguates | | MAME BIOS ZIPs | `contents` field documents inner structure | diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index 5658005e..5b43d469 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -47,13 +47,14 @@ hashed with MD5, and compared case-insensitively against the expected value. If the `resolve_local_file` step already confirmed the MD5 match (status `md5_exact`), `verify.py` skips re-hashing and returns OK directly. -### Truncated MD5 (Batocera) +### MD5 shorter than 32 characters (Batocera) -Some entries in Batocera's system data carry 29-character MD5 strings instead of -the usual 32. Batocera's script compares the full string (`batocera-systems:1088`), so -those entries report the file as untested. `verify.py` compares the prefix instead, to -find the file the entry designates: if the expected hash is shorter than 32 characters, -the actual hash is compared against only its first N characters. +Batocera's script compares the full string (`batocera-systems:1088`), so an MD5 +shorter than 32 characters matches no file there. The Batocera scraper completes such +a value when exactly one MD5 in `database.json` starts with it +(`_resolve_truncated_md5` in `scripts/scraper/batocera_scraper.py`) and keeps it as +written otherwise. `resolve_local_file` matches a value still shorter than 32 +characters by prefix, and only for a file whose name the entry also names. ### md5_composite (Recalbox ZIP verification)