kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-23 20:29:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: keep recalbox composite md5 on archives

Browse Source 
Bios.cpp:64-68 hashes a .zip with Zip::Md5Composite, the md5 of its
members in name order. A profile's md5 on an archive is the container's,
so it neither pairs with, corrects nor fills that field, and an archive
addition with only a container md5 is left out.
This commit is contained in:
Abdessamad Derraz committed 2026-10-21 05:08:45 +02:00
1 parent f796155e1a
commit fbf7cb191b
4 files changed
+97 -41
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

exports/recalbox.json
+3 -35
View File
@@ -7,11 +7,11 @@
"files": [
"es_bios.xml"
],
"summary": "357 kept, 88 added, 4 hashes corrected, 129 requirements corrected, 527 additions left out, 102 entries found by content not listed",
"summary": "357 kept, 87 added, 2 hashes corrected, 129 requirements corrected, 528 additions left out, 102 entries found by content not listed",
"counts": {
"kept": 357,
"added": 88,
"not_added": 527,
"added": 87,
"not_added": 528,
"not_kept": 0
},
"changes": [
@@ -102,18 +102,6 @@
"required": false
}
},
{
"kind": "added",
"system": "amiga600",
"name": "boot-data.zip",
"destination": "whdboot/boot-data.zip",
"field": null,
"old": null,
"new": {
"md5": "9e638f3d531eb7cd82594f49a4cf03b7",
"required": false
}
},
{
"kind": "added",
"system": "amiga600",
@@ -1953,15 +1941,6 @@
"old": true,
"new": false
},
{
"kind": "hash_corrected",
"system": "neogeocd",
"name": "neocdz.zip",
"destination": "neocdz.zip",
"field": "md5",
"old": "f39572af7584cb5b3f70ae8cc848aba2,389dbeb0cb857c130414747462cf9d71,c85b8132799f1b2ad3754a97947809d2,b9ab07a79997fc044ca164fc72cd5680",
"new": "b31a246b69493c35194fe0bac120dc37"
},
{
"kind": "requirement_corrected",
"system": "neogeocd",
@@ -1971,15 +1950,6 @@
"old": true,
"new": false
},
{
"kind": "hash_corrected",
"system": "neogeocd",
"name": "neocd.zip",
"destination": "geolith/neocd.zip",
"field": "md5",
"old": "3711783a59ef875c31076512e3f9f0e5",
"new": "006a881532f2caffc3fe9fb5593f2ed5"
},
{
"kind": "requirement_corrected",
"system": "neogeocd",
@@ -2270,8 +2240,6 @@
}
],
"hashes_corrected": [
"neogeocd/neocdz.zip md5",
"neogeocd/neocd.zip md5",
"ti994a/TI-994A.ctg md5",
"ti994a/ti-disk.ctg md5"
],
exports/recalbox/es_bios.xml
+2 -3
View File
@@ -12,7 +12,6 @@
<bios path="whdboot/WHDLoad" md5="6670a9571f235ec31b8bbd48fc18f428" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/JST" md5="2a3a61bbd7c351751506e22a4354b76a" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/AmiQuit" md5="36ab26d3b855e634fdda062c052cd52b" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/boot-data.zip" md5="9e638f3d531eb7cd82594f49a4cf03b7" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/game-data/whdload_db.json" md5="14d797815aa9bbc79fe284b244ea75b1" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/save-data/Kickstarts/kick33180.A500.RTB" md5="c993feb1eeddec1d2e687aa514231028" core="libretro/amiberry" mandatory="false" />
<bios path="whdboot/save-data/Kickstarts/kick34005.A500.RTB" md5="7eb13d974af0992d672b67c317a7e0a1" core="libretro/amiberry" mandatory="false" />
@@ -616,7 +615,7 @@
b4519df27a352c2a6e42e06d31330d91,bf00272e7150b31156a8995d60bf185d,
ca66fd463ef7dcab492a6de8ce5f45eb,f4a125538dfd7a8044e025fc5188fb88,
a3b2c9a69c28ad0bd4ea07877b744bbe,00dad01abdbf8ea9e79ad2fe11bdb182" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
<bios path="neocdz.zip" md5="b31a246b69493c35194fe0bac120dc37" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
<bios path="neocdz.zip" md5="f39572af7584cb5b3f70ae8cc848aba2,389dbeb0cb857c130414747462cf9d71,c85b8132799f1b2ad3754a97947809d2,b9ab07a79997fc044ca164fc72cd5680" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
<bios path="neocd/neocd.bin" md5="f39572af7584cb5b3f70ae8cc848aba2" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="CDZ BIOS (MAME)" />
<bios path="neocd/uni-bioscd.rom" md5="08ca8b2dba6662e8024f9e789711c6fc" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Universe BIOS CD 3.3" />
<bios path="neocd/top-sp1.bin" md5="122aee210324c72e8a11116e6ef9c0d0" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Top Loader BIOS (MAME)" />
@@ -629,7 +628,7 @@
<bios path="neocd/neocd_f.rom" md5="8834880c33164ccbe6476b559f3e37de" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Front Loader BIOS" />
<bios path="neocd/000-lo.lo" md5="fc7599f3f871578fe9a0453662d1c966" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="ZOOM Rom" />
<bios path="neocd/ng-lo.rom" md5="e255264d85d5765013b1b2fa8109dd53" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="ZOOM Rom" />
<bios path="geolith/neocd.zip" md5="006a881532f2caffc3fe9fb5593f2ed5" core="libretro/geolith" mandatory="false" hashMatchMandatory="false" note="Neo-Geo CD Bios" />
<bios path="geolith/neocd.zip" md5="3711783a59ef875c31076512e3f9f0e5" core="libretro/geolith" mandatory="false" hashMatchMandatory="false" note="Neo-Geo CD Bios" />
<bios path="geolith/neocdz.zip" md5="724bc64e63d073cf81f986adfb1f1b78" core="libretro/geolith" mandatory="true" hashMatchMandatory="false" note="Neo-Geo CDZ Bios" />
</system>
<system fullname="Texas Instrument TI-99/4A" platform="ti994a">
scripts/exporter/baseline.py
+35 -3
View File
@@ -61,6 +61,20 @@ def checked_hashes(entry: dict | None) -> dict | None:
return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds}
def comparable(entry: dict | None) -> dict:
"""The entry with only the hashes the truth can be compared with.
Recalbox hashes a .zip by its members (Bios.cpp:64-68 calls
Zip::Md5Composite, which feeds each member's bytes in name order into
one md5), never by the container's bytes. Its md5 on an archive is that
composite, a digest no profile declares: read as a container md5 it was
replaced by geolith's, which Recalbox never computes.
"""
if not entry or entry.get("zip_digest") != "composite":
return entry or {}
return {k: v for k, v in entry.items() if k != "md5"}
def is_label(entry: dict) -> bool:
"""A name only content-scanning cores give: no core opens that file.
@@ -104,6 +118,10 @@ class NativeFile:
# For an addition, each core that wants it as (runs standalone, name the
# platform gives it), empty when no core the platform runs is named.
runners: list[tuple[bool, str]] = field(default_factory=list)
# The platform reads this file's md5 as a composite of its members
# (Recalbox on a .zip): the truth's md5 describes the container and
# can be neither compared with it nor written in its place.
composite_md5: bool = False
@property
def origin(self) -> str:
@@ -129,7 +147,9 @@ class NativeFile:
truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and
reading it as a contradiction emptied the md5 the frontend checks.
"""
truth, plat = _field_sets(self._checked_truth), _field_sets(self.platform)
truth, plat = _field_sets(self._checked_truth), _field_sets(comparable(self.platform))
if self.composite_md5:
truth["md5"] = set()
if not any(truth.values()):
return "platform"
if not any(plat.values()):
@@ -152,6 +172,8 @@ class NativeFile:
the divergence is recorded, never silently merged: an emulator that
rejects a file will reject it whatever the platform declares.
"""
if field_name == "md5" and self.composite_md5:
return _hash_values(self.platform or {}, field_name)
truth_values = _hash_values(self._checked_truth, field_name)
platform_values = _hash_values(self.platform or {}, field_name)
source = self._identity_source()
@@ -186,6 +208,9 @@ class NativeFile:
chosen among those the truth accepts. A field only the truth names
is written when it names one value, which is then that dump's.
"""
if field_name == "md5" and self.composite_md5:
values = _hash_values(self.platform or {}, field_name)
return values[0] if values else ""
source = self._identity_source()
truth_values = _hash_values(self._checked_truth, field_name)
platform_values = _hash_values(self.platform or {}, field_name)
@@ -365,11 +390,13 @@ def by_hash(truth_entry: dict, platform_entry: dict) -> bool:
ours = {v for f in HASH_FIELDS for v in _hash_values(truth_entry, f)}
if not ours:
return False
return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(platform_entry, f)})
theirs = comparable(platform_entry)
return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(theirs, f)})
def _contradicts(truth_entry: dict, platform_entry: dict) -> bool:
"""A hash both entries declare, without a value in common."""
platform_entry = comparable(platform_entry)
return any(
(ours := set(_hash_values(truth_entry, f)))
and (theirs := set(_hash_values(platform_entry, f)))
@@ -445,6 +472,7 @@ def build_native_model(
systems: dict[str, NativeSystem] = {}
scraped_systems = (scraped or {}).get("systems", {})
composite_zips = (scraped or {}).get("zip_digest") == "composite"
# Pass 1: the platform's own file, grouped as the platform groups it.
for sys_key, sys_data in scraped_systems.items():
@@ -467,6 +495,7 @@ def build_native_model(
destination=destination,
native_system=native_id,
platform=file_entry,
composite_md5=file_entry.get("zip_digest") == "composite",
)
)
report.files_kept += 1
@@ -545,7 +574,9 @@ def build_native_model(
checked = checked_hashes(truth_entry) or {}
for field_name in HASH_FIELDS:
ours = set(_hash_values(checked, field_name))
theirs = set(_hash_values(matched.platform or {}, field_name))
if field_name == "md5" and matched.composite_md5:
continue
theirs = set(_hash_values(comparable(matched.platform), field_name))
if ours and theirs and not ours & theirs:
matched.corrections.append(field_name)
report.hashes_corrected.append(
@@ -602,6 +633,7 @@ def build_native_model(
native_system=native_id,
truth=truth_entry,
foreign_system=native_id not in platform_ids,
composite_md5=composite_zips and name.lower().endswith(".zip"),
)
)
report.files_added += 1
tests/test_exporters.py
+57
View File
@@ -43,6 +43,7 @@ from export_native import ( # noqa: E402
SOURCES_INDEX,
collect_originals,
fetch,
landed_changes,
pinned_base,
source_key,
)
@@ -1084,6 +1085,62 @@ class RecalboxExport(unittest.TestCase):
}
self.assertEqual(blocks, {pentagon: ["3" * 32], scorpion: [new]})
COMPOSITE_SCRAPED = {
"zip_digest": "composite",
"systems": {"snk-neogeo-cd": {"native_id": "neogeocd", "files": [{
"name": "neocdz.zip", "destination": "neocdz.zip", "md5": "a" * 32,
"native_system": "neogeocd", "core": "libretro/fbneo",
"required": True, "zip_digest": "composite",
}]}},
}
def test_a_composite_md5_is_not_replaced_by_a_container_md5(self):
"""Recalbox hashes a .zip by its members (Bios.cpp:64-68,
Zip::Md5Composite): the truth's container md5 is another digest."""
truth = {"systems": {"snk-neogeo-cd": {"files": [
{"name": "neocdz.zip", "md5": "b" * 32, "required": True, "_cores": ["geolith"]},
{"name": "boot-data.zip", "md5": "c" * 32, "required": False, "_cores": ["geolith"]},
]}}}
systems, report = build_native_model(truth, self.COMPOSITE_SCRAPED)
exporter = discover_exporters()["recalbox"]()
produced = exporter.render(systems, report, {}, None)["es_bios.xml"]
md5_of = {b.get("path"): b.get("md5") for b in ET.fromstring(produced).iter("bios")}
self.assertEqual(md5_of, {"neocdz.zip": "a" * 32})
self.assertEqual(landed_changes(exporter, systems).records, [])
def test_a_container_md5_never_pairs_with_a_composite(self):
"""neocd.bin's md5 equals the composite of a neocdz.zip: by hash the
two would be read as one renamed file."""
truth = {"systems": {"snk-neogeo-cd": {"files": [
{"name": "neocd.bin", "md5": "a" * 32, "required": True, "_cores": ["neocd"]},
]}}}
systems, _ = build_native_model(truth, self.COMPOSITE_SCRAPED)
declared = next(fe for s in systems.values() for fe in s.files if fe.platform)
self.assertIsNone(declared.truth)
def test_the_live_export_keeps_every_zip_md5(self):
"""No element of es_bios.xml names a .zip under a new md5: none can
be stated as the composite Recalbox computes."""
path = REPO_ROOT / "exports" / "recalbox" / "es_bios.xml"
original = UPSTREAM_CACHE / "recalbox" / "es_bios.xml"
if not original.is_file():
self.skipTest("no cached original for recalbox")
def zips(text: str) -> dict[tuple[str, str], str]:
return {
(s.get("platform"), b.get("path")): ",".join(sorted(
v.strip().lower() for v in b.get("md5", "").split(",")
))
for s in ET.fromstring(text).iter("system")
for b in s.iter("bios")
if b.get("path", "").lower().endswith(".zip")
}
before = zips(original.read_text(encoding="utf-8"))
after = zips(path.read_text(encoding="utf-8"))
self.assertEqual({k: v for k, v in after.items() if k in before}, before)
self.assertEqual(set(after) - set(before), set())
def test_a_truncated_export_is_reported(self):
exporter, systems, _ = self._render()
empty = {