From fbf7cb191b441af035ac3f166344639274d18add Mon Sep 17 00:00:00 2001
From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com>
Date: Wed, 21 Oct 2026 05:08:45 +0200
Subject: [PATCH] fix: keep recalbox composite md5 on archives
Bios.cpp:64-68 hashes a .zip with Zip::Md5Composite, the md5 of its
members in name order. A profile's md5 on an archive is the container's,
so it neither pairs with, corrects nor fills that field, and an archive
addition with only a container md5 is left out.
---
exports/recalbox.json | 38 ++----------------------
exports/recalbox/es_bios.xml | 5 ++--
scripts/exporter/baseline.py | 38 ++++++++++++++++++++++--
tests/test_exporters.py | 57 ++++++++++++++++++++++++++++++++++++
4 files changed, 97 insertions(+), 41 deletions(-)
diff --git a/exports/recalbox.json b/exports/recalbox.json
index a342a215..b5f738f4 100644
--- a/exports/recalbox.json
+++ b/exports/recalbox.json
@@ -7,11 +7,11 @@
"files": [
"es_bios.xml"
],
- "summary": "357 kept, 88 added, 4 hashes corrected, 129 requirements corrected, 527 additions left out, 102 entries found by content not listed",
+ "summary": "357 kept, 87 added, 2 hashes corrected, 129 requirements corrected, 528 additions left out, 102 entries found by content not listed",
"counts": {
"kept": 357,
- "added": 88,
- "not_added": 527,
+ "added": 87,
+ "not_added": 528,
"not_kept": 0
},
"changes": [
@@ -102,18 +102,6 @@
"required": false
}
},
- {
- "kind": "added",
- "system": "amiga600",
- "name": "boot-data.zip",
- "destination": "whdboot/boot-data.zip",
- "field": null,
- "old": null,
- "new": {
- "md5": "9e638f3d531eb7cd82594f49a4cf03b7",
- "required": false
- }
- },
{
"kind": "added",
"system": "amiga600",
@@ -1953,15 +1941,6 @@
"old": true,
"new": false
},
- {
- "kind": "hash_corrected",
- "system": "neogeocd",
- "name": "neocdz.zip",
- "destination": "neocdz.zip",
- "field": "md5",
- "old": "f39572af7584cb5b3f70ae8cc848aba2,389dbeb0cb857c130414747462cf9d71,c85b8132799f1b2ad3754a97947809d2,b9ab07a79997fc044ca164fc72cd5680",
- "new": "b31a246b69493c35194fe0bac120dc37"
- },
{
"kind": "requirement_corrected",
"system": "neogeocd",
@@ -1971,15 +1950,6 @@
"old": true,
"new": false
},
- {
- "kind": "hash_corrected",
- "system": "neogeocd",
- "name": "neocd.zip",
- "destination": "geolith/neocd.zip",
- "field": "md5",
- "old": "3711783a59ef875c31076512e3f9f0e5",
- "new": "006a881532f2caffc3fe9fb5593f2ed5"
- },
{
"kind": "requirement_corrected",
"system": "neogeocd",
@@ -2270,8 +2240,6 @@
}
],
"hashes_corrected": [
- "neogeocd/neocdz.zip md5",
- "neogeocd/neocd.zip md5",
"ti994a/TI-994A.ctg md5",
"ti994a/ti-disk.ctg md5"
],
diff --git a/exports/recalbox/es_bios.xml b/exports/recalbox/es_bios.xml
index 07514c27..ee5b9e89 100644
--- a/exports/recalbox/es_bios.xml
+++ b/exports/recalbox/es_bios.xml
@@ -12,7 +12,6 @@
-
@@ -616,7 +615,7 @@
b4519df27a352c2a6e42e06d31330d91,bf00272e7150b31156a8995d60bf185d,
ca66fd463ef7dcab492a6de8ce5f45eb,f4a125538dfd7a8044e025fc5188fb88,
a3b2c9a69c28ad0bd4ea07877b744bbe,00dad01abdbf8ea9e79ad2fe11bdb182" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
-
+
@@ -629,7 +628,7 @@
-
+
diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py
index 2f79fab6..f55bfb1a 100644
--- a/scripts/exporter/baseline.py
+++ b/scripts/exporter/baseline.py
@@ -61,6 +61,20 @@ def checked_hashes(entry: dict | None) -> dict | None:
return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds}
+def comparable(entry: dict | None) -> dict:
+ """The entry with only the hashes the truth can be compared with.
+
+ Recalbox hashes a .zip by its members (Bios.cpp:64-68 calls
+ Zip::Md5Composite, which feeds each member's bytes in name order into
+ one md5), never by the container's bytes. Its md5 on an archive is that
+ composite, a digest no profile declares: read as a container md5 it was
+ replaced by geolith's, which Recalbox never computes.
+ """
+ if not entry or entry.get("zip_digest") != "composite":
+ return entry or {}
+ return {k: v for k, v in entry.items() if k != "md5"}
+
+
def is_label(entry: dict) -> bool:
"""A name only content-scanning cores give: no core opens that file.
@@ -104,6 +118,10 @@ class NativeFile:
# For an addition, each core that wants it as (runs standalone, name the
# platform gives it), empty when no core the platform runs is named.
runners: list[tuple[bool, str]] = field(default_factory=list)
+ # The platform reads this file's md5 as a composite of its members
+ # (Recalbox on a .zip): the truth's md5 describes the container and
+ # can be neither compared with it nor written in its place.
+ composite_md5: bool = False
@property
def origin(self) -> str:
@@ -129,7 +147,9 @@ class NativeFile:
truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and
reading it as a contradiction emptied the md5 the frontend checks.
"""
- truth, plat = _field_sets(self._checked_truth), _field_sets(self.platform)
+ truth, plat = _field_sets(self._checked_truth), _field_sets(comparable(self.platform))
+ if self.composite_md5:
+ truth["md5"] = set()
if not any(truth.values()):
return "platform"
if not any(plat.values()):
@@ -152,6 +172,8 @@ class NativeFile:
the divergence is recorded, never silently merged: an emulator that
rejects a file will reject it whatever the platform declares.
"""
+ if field_name == "md5" and self.composite_md5:
+ return _hash_values(self.platform or {}, field_name)
truth_values = _hash_values(self._checked_truth, field_name)
platform_values = _hash_values(self.platform or {}, field_name)
source = self._identity_source()
@@ -186,6 +208,9 @@ class NativeFile:
chosen among those the truth accepts. A field only the truth names
is written when it names one value, which is then that dump's.
"""
+ if field_name == "md5" and self.composite_md5:
+ values = _hash_values(self.platform or {}, field_name)
+ return values[0] if values else ""
source = self._identity_source()
truth_values = _hash_values(self._checked_truth, field_name)
platform_values = _hash_values(self.platform or {}, field_name)
@@ -365,11 +390,13 @@ def by_hash(truth_entry: dict, platform_entry: dict) -> bool:
ours = {v for f in HASH_FIELDS for v in _hash_values(truth_entry, f)}
if not ours:
return False
- return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(platform_entry, f)})
+ theirs = comparable(platform_entry)
+ return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(theirs, f)})
def _contradicts(truth_entry: dict, platform_entry: dict) -> bool:
"""A hash both entries declare, without a value in common."""
+ platform_entry = comparable(platform_entry)
return any(
(ours := set(_hash_values(truth_entry, f)))
and (theirs := set(_hash_values(platform_entry, f)))
@@ -445,6 +472,7 @@ def build_native_model(
systems: dict[str, NativeSystem] = {}
scraped_systems = (scraped or {}).get("systems", {})
+ composite_zips = (scraped or {}).get("zip_digest") == "composite"
# Pass 1: the platform's own file, grouped as the platform groups it.
for sys_key, sys_data in scraped_systems.items():
@@ -467,6 +495,7 @@ def build_native_model(
destination=destination,
native_system=native_id,
platform=file_entry,
+ composite_md5=file_entry.get("zip_digest") == "composite",
)
)
report.files_kept += 1
@@ -545,7 +574,9 @@ def build_native_model(
checked = checked_hashes(truth_entry) or {}
for field_name in HASH_FIELDS:
ours = set(_hash_values(checked, field_name))
- theirs = set(_hash_values(matched.platform or {}, field_name))
+ if field_name == "md5" and matched.composite_md5:
+ continue
+ theirs = set(_hash_values(comparable(matched.platform), field_name))
if ours and theirs and not ours & theirs:
matched.corrections.append(field_name)
report.hashes_corrected.append(
@@ -602,6 +633,7 @@ def build_native_model(
native_system=native_id,
truth=truth_entry,
foreign_system=native_id not in platform_ids,
+ composite_md5=composite_zips and name.lower().endswith(".zip"),
)
)
report.files_added += 1
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 402b2e66..904735b2 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -43,6 +43,7 @@ from export_native import ( # noqa: E402
SOURCES_INDEX,
collect_originals,
fetch,
+ landed_changes,
pinned_base,
source_key,
)
@@ -1084,6 +1085,62 @@ class RecalboxExport(unittest.TestCase):
}
self.assertEqual(blocks, {pentagon: ["3" * 32], scorpion: [new]})
+ COMPOSITE_SCRAPED = {
+ "zip_digest": "composite",
+ "systems": {"snk-neogeo-cd": {"native_id": "neogeocd", "files": [{
+ "name": "neocdz.zip", "destination": "neocdz.zip", "md5": "a" * 32,
+ "native_system": "neogeocd", "core": "libretro/fbneo",
+ "required": True, "zip_digest": "composite",
+ }]}},
+ }
+
+ def test_a_composite_md5_is_not_replaced_by_a_container_md5(self):
+ """Recalbox hashes a .zip by its members (Bios.cpp:64-68,
+ Zip::Md5Composite): the truth's container md5 is another digest."""
+ truth = {"systems": {"snk-neogeo-cd": {"files": [
+ {"name": "neocdz.zip", "md5": "b" * 32, "required": True, "_cores": ["geolith"]},
+ {"name": "boot-data.zip", "md5": "c" * 32, "required": False, "_cores": ["geolith"]},
+ ]}}}
+ systems, report = build_native_model(truth, self.COMPOSITE_SCRAPED)
+ exporter = discover_exporters()["recalbox"]()
+ produced = exporter.render(systems, report, {}, None)["es_bios.xml"]
+ md5_of = {b.get("path"): b.get("md5") for b in ET.fromstring(produced).iter("bios")}
+ self.assertEqual(md5_of, {"neocdz.zip": "a" * 32})
+ self.assertEqual(landed_changes(exporter, systems).records, [])
+
+ def test_a_container_md5_never_pairs_with_a_composite(self):
+ """neocd.bin's md5 equals the composite of a neocdz.zip: by hash the
+ two would be read as one renamed file."""
+ truth = {"systems": {"snk-neogeo-cd": {"files": [
+ {"name": "neocd.bin", "md5": "a" * 32, "required": True, "_cores": ["neocd"]},
+ ]}}}
+ systems, _ = build_native_model(truth, self.COMPOSITE_SCRAPED)
+ declared = next(fe for s in systems.values() for fe in s.files if fe.platform)
+ self.assertIsNone(declared.truth)
+
+ def test_the_live_export_keeps_every_zip_md5(self):
+ """No element of es_bios.xml names a .zip under a new md5: none can
+ be stated as the composite Recalbox computes."""
+ path = REPO_ROOT / "exports" / "recalbox" / "es_bios.xml"
+ original = UPSTREAM_CACHE / "recalbox" / "es_bios.xml"
+ if not original.is_file():
+ self.skipTest("no cached original for recalbox")
+
+ def zips(text: str) -> dict[tuple[str, str], str]:
+ return {
+ (s.get("platform"), b.get("path")): ",".join(sorted(
+ v.strip().lower() for v in b.get("md5", "").split(",")
+ ))
+ for s in ET.fromstring(text).iter("system")
+ for b in s.iter("bios")
+ if b.get("path", "").lower().endswith(".zip")
+ }
+
+ before = zips(original.read_text(encoding="utf-8"))
+ after = zips(path.read_text(encoding="utf-8"))
+ self.assertEqual({k: v for k, v in after.items() if k in before}, before)
+ self.assertEqual(set(after) - set(before), set())
+
def test_a_truncated_export_is_reported(self):
exporter, systems, _ = self._render()
empty = {