mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
docs: describe short md5 handling as the code does
No platform YAML holds an md5 shorter than 32 characters: the Batocera scraper completes one when a single collection md5 starts with it. The pages said some entries carry 29 characters and that scrapers store them as-is.
This commit is contained in:
1 parent
a6d37e40ae
commit
d8e9ad23d3
8 files changed
+44
-15
No files matched your search
@@ -314,8 +314,8 @@ def _hash_matches(declared: str, actual: str) -> bool:
|
||||
"""Compare a declared hash value against an actual hex digest.
|
||||
|
||||
Handles comma-separated multi-hash lists and uppercase (Recalbox)
|
||||
and truncated MD5s (Batocera 29-char): a declared value shorter
|
||||
than the digest matches by prefix.
|
||||
and MD5s shorter than 32 characters: a declared value shorter than
|
||||
the digest matches by prefix.
|
||||
"""
|
||||
actual = actual.lower()
|
||||
for cand in declared.split(","):
|
||||
|
||||
@@ -497,5 +497,31 @@ class EmulatorVerboseSample(unittest.TestCase):
|
||||
self.assertIn(line, printed)
|
||||
|
||||
|
||||
class ShortMd5Handling(unittest.TestCase):
|
||||
"""Three pages said some Batocera entries carry 29-character MD5s and show
|
||||
as untested. No platform YAML holds one, since the Batocera scraper
|
||||
completes them, and adding-a-scraper.md said to store them as-is."""
|
||||
|
||||
def test_no_page_claims_short_md5_entries(self):
|
||||
offenders = [
|
||||
page.name for page in PUBLISHED
|
||||
if re.search(r"(?i)29[- ]char|truncated md5 hashes in some entries|store the hash as-is",
|
||||
page.read_text(encoding="utf-8"))
|
||||
]
|
||||
self.assertEqual(offenders, [])
|
||||
|
||||
def test_the_page_names_what_the_code_does(self):
|
||||
import common
|
||||
from scraper import batocera_scraper
|
||||
|
||||
scraper = inspect.getsource(batocera_scraper)
|
||||
self.assertIn("md5 = _resolve_truncated_md5(", scraper)
|
||||
resolver = inspect.getsource(common.resolve_local_file)
|
||||
self.assertIn("if len(md5_candidate) < 32:", resolver)
|
||||
self.assertIn("_md5_name_ok(path)", resolver)
|
||||
section = _section(_page("verification-modes.md"), "MD5 shorter than 32 characters (Batocera)")
|
||||
self.assertIn("_resolve_truncated_md5", section)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+2
-2
@@ -265,7 +265,7 @@ class TestE2E(unittest.TestCase):
|
||||
).hexdigest() # sorted: a.rom, b.rom
|
||||
multi_wrong = "0000000000000000000000000000000"
|
||||
multi_right = f["multi.zip"]["inner_md5s"]["rom.bin"]
|
||||
truncated_md5 = f["truncated.bin"]["md5"][:29] # Batocera 29-char
|
||||
truncated_md5 = f["truncated.bin"]["md5"][:29] # shorter than 32 characters
|
||||
|
||||
config = {
|
||||
"platform": "TestMD5",
|
||||
@@ -347,7 +347,7 @@ class TestE2E(unittest.TestCase):
|
||||
"zipped_file": "rom.bin",
|
||||
"required": True,
|
||||
},
|
||||
# Truncated MD5 (Batocera 29 chars)
|
||||
# MD5 shorter than 32 characters
|
||||
{
|
||||
"name": "truncated.bin",
|
||||
"destination": "truncated.bin",
|
||||
|
||||
@@ -86,7 +86,7 @@ class DeclaredHashVerdict(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_a_truncated_md5_still_decides(self):
|
||||
"""Batocera publishes 29-character prefixes, which are still evidence."""
|
||||
"""An MD5 shorter than 32 characters is still evidence, by prefix."""
|
||||
self.assertEqual(
|
||||
self.verdict(self.right, md5_list=[self.d["md5"][:29]]),
|
||||
"data_dir_hash_exact",
|
||||
|
||||
@@ -118,7 +118,7 @@ class ShippedFilesSatisfyTheirDeclaration(unittest.TestCase):
|
||||
digests["md5"].append(common.md5_composite(str(blob_path)))
|
||||
except (zipfile.BadZipFile, OSError):
|
||||
pass
|
||||
# Batocera publishes 29-character prefixes, so compare by prefix.
|
||||
# A declared MD5 can be shorter than 32 characters: compare by prefix.
|
||||
satisfied = any(
|
||||
all(
|
||||
any(
|
||||
|
||||
@@ -206,8 +206,10 @@ System ID consistency matters for cross-platform operations. The same console mu
|
||||
- Handle missing hashes gracefully (set to `None`, not empty string).
|
||||
- Some platforms provide multiple hash types per entry. Populate whichever fields
|
||||
are available.
|
||||
- Batocera uses 29-character truncated MD5 hashes in some entries. The resolution
|
||||
layer handles prefix matching, but the scraper should store the hash as-is.
|
||||
- An MD5 shorter than 32 characters is completed when exactly one MD5 in
|
||||
`database.json` starts with it, as the Batocera scraper does
|
||||
(`_resolve_truncated_md5`), and kept as written otherwise. The resolution layer
|
||||
matches what remains by prefix.
|
||||
|
||||
Scraped data reflects what the upstream declares, which may not match reality. The scraper's job is faithful transcription of upstream data, not correction. Corrections happen in the emulator profiles (source-verified) and in `_shared.yml` (curated). If a scraper meets a hash or filename that differs from the emulator source, log a warning and still include the upstream value. The divergence will surface during truth diffing.
|
||||
|
||||
|
||||
@@ -372,7 +372,7 @@ platform's own verification mode.
|
||||
|
||||
| Case | Handling |
|
||||
|------|---------|
|
||||
| Batocera truncated MD5 (29 chars) | prefix match in resolution |
|
||||
| MD5 shorter than 32 characters | completed by the Batocera scraper when one MD5 in the collection starts with it, matched by prefix in resolution |
|
||||
| `zippedFile` entries | MD5 is of the ROM inside the ZIP, not the ZIP itself |
|
||||
| Regional variants (same filename) | `by_path_suffix` index disambiguates |
|
||||
| MAME BIOS ZIPs | `contents` field documents inner structure |
|
||||
|
||||
@@ -47,13 +47,14 @@ hashed with MD5, and compared case-insensitively against the expected value.
|
||||
If the `resolve_local_file` step already confirmed the MD5 match (status `md5_exact`),
|
||||
`verify.py` skips re-hashing and returns OK directly.
|
||||
|
||||
### Truncated MD5 (Batocera)
|
||||
### MD5 shorter than 32 characters (Batocera)
|
||||
|
||||
Some entries in Batocera's system data carry 29-character MD5 strings instead of
|
||||
the usual 32. Batocera's script compares the full string (`batocera-systems:1088`), so
|
||||
those entries report the file as untested. `verify.py` compares the prefix instead, to
|
||||
find the file the entry designates: if the expected hash is shorter than 32 characters,
|
||||
the actual hash is compared against only its first N characters.
|
||||
Batocera's script compares the full string (`batocera-systems:1088`), so an MD5
|
||||
shorter than 32 characters matches no file there. The Batocera scraper completes such
|
||||
a value when exactly one MD5 in `database.json` starts with it
|
||||
(`_resolve_truncated_md5` in `scripts/scraper/batocera_scraper.py`) and keeps it as
|
||||
written otherwise. `resolve_local_file` matches a value still shorter than 32
|
||||
characters by prefix, and only for a file whose name the entry also names.
|
||||
|
||||
### md5_composite (Recalbox ZIP verification)
|
||||
|
||||
|
||||
Reference in new issue
Block a user