fix: keep recalbox composite md5 on archives

Bios.cpp:64-68 hashes a .zip with Zip::Md5Composite, the md5 of its
members in name order. A profile's md5 on an archive is the container's,
so it neither pairs with, corrects nor fills that field, and an archive
addition with only a container md5 is left out.

exports/recalbox.json

@@ -7,11 +7,11 @@
   "files": [
     "es_bios.xml"
   ],
-  "summary": "357 kept, 88 added, 4 hashes corrected, 129 requirements corrected, 527 additions left out, 102 entries found by content not listed",
+  "summary": "357 kept, 87 added, 2 hashes corrected, 129 requirements corrected, 528 additions left out, 102 entries found by content not listed",
   "counts": {
     "kept": 357,
-    "added": 88,
+    "added": 87,
-    "not_added": 527,
+    "not_added": 528,
     "not_kept": 0
   },
   "changes": [
@@ -102,18 +102,6 @@
         "required": false
       }
     },
-    {
-      "kind": "added",
-      "system": "amiga600",
-      "name": "boot-data.zip",
-      "destination": "whdboot/boot-data.zip",
-      "field": null,
-      "old": null,
-      "new": {
-        "md5": "9e638f3d531eb7cd82594f49a4cf03b7",
-        "required": false
-      }
-    },
     {
       "kind": "added",
       "system": "amiga600",
@@ -1953,15 +1941,6 @@
       "old": true,
       "new": false
     },
-    {
-      "kind": "hash_corrected",
-      "system": "neogeocd",
-      "name": "neocdz.zip",
-      "destination": "neocdz.zip",
-      "field": "md5",
-      "old": "f39572af7584cb5b3f70ae8cc848aba2,389dbeb0cb857c130414747462cf9d71,c85b8132799f1b2ad3754a97947809d2,b9ab07a79997fc044ca164fc72cd5680",
-      "new": "b31a246b69493c35194fe0bac120dc37"
-    },
     {
       "kind": "requirement_corrected",
       "system": "neogeocd",
@@ -1971,15 +1950,6 @@
       "old": true,
       "new": false
     },
-    {
-      "kind": "hash_corrected",
-      "system": "neogeocd",
-      "name": "neocd.zip",
-      "destination": "geolith/neocd.zip",
-      "field": "md5",
-      "old": "3711783a59ef875c31076512e3f9f0e5",
-      "new": "006a881532f2caffc3fe9fb5593f2ed5"
-    },
     {
       "kind": "requirement_corrected",
       "system": "neogeocd",
@@ -2270,8 +2240,6 @@
     }
   ],
   "hashes_corrected": [
-    "neogeocd/neocdz.zip md5",
-    "neogeocd/neocd.zip md5",
     "ti994a/TI-994A.ctg md5",
     "ti994a/ti-disk.ctg md5"
   ],

exports/recalbox/es_bios.xml

@@ -12,7 +12,6 @@
     <bios path="whdboot/WHDLoad" md5="6670a9571f235ec31b8bbd48fc18f428" core="libretro/amiberry" mandatory="false" />
     <bios path="whdboot/JST" md5="2a3a61bbd7c351751506e22a4354b76a" core="libretro/amiberry" mandatory="false" />
     <bios path="whdboot/AmiQuit" md5="36ab26d3b855e634fdda062c052cd52b" core="libretro/amiberry" mandatory="false" />
-    <bios path="whdboot/boot-data.zip" md5="9e638f3d531eb7cd82594f49a4cf03b7" core="libretro/amiberry" mandatory="false" />
     <bios path="whdboot/game-data/whdload_db.json" md5="14d797815aa9bbc79fe284b244ea75b1" core="libretro/amiberry" mandatory="false" />
     <bios path="whdboot/save-data/Kickstarts/kick33180.A500.RTB" md5="c993feb1eeddec1d2e687aa514231028" core="libretro/amiberry" mandatory="false" />
     <bios path="whdboot/save-data/Kickstarts/kick34005.A500.RTB" md5="7eb13d974af0992d672b67c317a7e0a1" core="libretro/amiberry" mandatory="false" />
@@ -616,7 +615,7 @@
                                  b4519df27a352c2a6e42e06d31330d91,bf00272e7150b31156a8995d60bf185d,
                                  ca66fd463ef7dcab492a6de8ce5f45eb,f4a125538dfd7a8044e025fc5188fb88,
                                  a3b2c9a69c28ad0bd4ea07877b744bbe,00dad01abdbf8ea9e79ad2fe11bdb182" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
-    <bios path="neocdz.zip" md5="b31a246b69493c35194fe0bac120dc37" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
+    <bios path="neocdz.zip" md5="f39572af7584cb5b3f70ae8cc848aba2,389dbeb0cb857c130414747462cf9d71,c85b8132799f1b2ad3754a97947809d2,b9ab07a79997fc044ca164fc72cd5680" hashMatchMandatory="false" core="libretro/fbneo,libretro/fba2x,libretro/mame2000,libretro/mame2003,libretro/mame2003-plus,libretro/mame2010,libretro/mame2015" />
     <bios path="neocd/neocd.bin"                 md5="f39572af7584cb5b3f70ae8cc848aba2" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="CDZ BIOS (MAME)" />
     <bios path="neocd/uni-bioscd.rom"            md5="08ca8b2dba6662e8024f9e789711c6fc" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Universe BIOS CD 3.3" />
     <bios path="neocd/top-sp1.bin"               md5="122aee210324c72e8a11116e6ef9c0d0" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Top Loader BIOS (MAME)" />
@@ -629,7 +628,7 @@
     <bios path="neocd/neocd_f.rom"               md5="8834880c33164ccbe6476b559f3e37de" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="Front Loader BIOS" />
     <bios path="neocd/000-lo.lo"                 md5="fc7599f3f871578fe9a0453662d1c966" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="ZOOM Rom" />
     <bios path="neocd/ng-lo.rom"                 md5="e255264d85d5765013b1b2fa8109dd53" core="libretro/neocd" mandatory="false" hashMatchMandatory="false" note="ZOOM Rom" />
-    <bios path="geolith/neocd.zip"               md5="006a881532f2caffc3fe9fb5593f2ed5" core="libretro/geolith" mandatory="false" hashMatchMandatory="false" note="Neo-Geo CD Bios" />
+    <bios path="geolith/neocd.zip"               md5="3711783a59ef875c31076512e3f9f0e5" core="libretro/geolith" mandatory="false" hashMatchMandatory="false" note="Neo-Geo CD Bios" />
     <bios path="geolith/neocdz.zip"              md5="724bc64e63d073cf81f986adfb1f1b78" core="libretro/geolith" mandatory="true" hashMatchMandatory="false" note="Neo-Geo CDZ Bios" />
   </system>
   <system fullname="Texas Instrument TI-99/4A" platform="ti994a">

scripts/exporter/baseline.py

@@ -61,6 +61,20 @@ def checked_hashes(entry: dict | None) -> dict | None:
     return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds}
 
 
+def comparable(entry: dict | None) -> dict:
+    """The entry with only the hashes the truth can be compared with.
+
+    Recalbox hashes a .zip by its members (Bios.cpp:64-68 calls
+    Zip::Md5Composite, which feeds each member's bytes in name order into
+    one md5), never by the container's bytes. Its md5 on an archive is that
+    composite, a digest no profile declares: read as a container md5 it was
+    replaced by geolith's, which Recalbox never computes.
+    """
+    if not entry or entry.get("zip_digest") != "composite":
+        return entry or {}
+    return {k: v for k, v in entry.items() if k != "md5"}
+
+
 def is_label(entry: dict) -> bool:
     """A name only content-scanning cores give: no core opens that file.
 
@@ -104,6 +118,10 @@ class NativeFile:
     # For an addition, each core that wants it as (runs standalone, name the
     # platform gives it), empty when no core the platform runs is named.
     runners: list[tuple[bool, str]] = field(default_factory=list)
+    # The platform reads this file's md5 as a composite of its members
+    # (Recalbox on a .zip): the truth's md5 describes the container and
+    # can be neither compared with it nor written in its place.
+    composite_md5: bool = False
 
     @property
     def origin(self) -> str:
@@ -129,7 +147,9 @@ class NativeFile:
         truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and
         reading it as a contradiction emptied the md5 the frontend checks.
         """
-        truth, plat = _field_sets(self._checked_truth), _field_sets(self.platform)
+        truth, plat = _field_sets(self._checked_truth), _field_sets(comparable(self.platform))
+        if self.composite_md5:
+            truth["md5"] = set()
         if not any(truth.values()):
             return "platform"
         if not any(plat.values()):
@@ -152,6 +172,8 @@ class NativeFile:
         the divergence is recorded, never silently merged: an emulator that
         rejects a file will reject it whatever the platform declares.
         """
+        if field_name == "md5" and self.composite_md5:
+            return _hash_values(self.platform or {}, field_name)
         truth_values = _hash_values(self._checked_truth, field_name)
         platform_values = _hash_values(self.platform or {}, field_name)
         source = self._identity_source()
@@ -186,6 +208,9 @@ class NativeFile:
         chosen among those the truth accepts. A field only the truth names
         is written when it names one value, which is then that dump's.
         """
+        if field_name == "md5" and self.composite_md5:
+            values = _hash_values(self.platform or {}, field_name)
+            return values[0] if values else ""
         source = self._identity_source()
         truth_values = _hash_values(self._checked_truth, field_name)
         platform_values = _hash_values(self.platform or {}, field_name)
@@ -365,11 +390,13 @@ def by_hash(truth_entry: dict, platform_entry: dict) -> bool:
     ours = {v for f in HASH_FIELDS for v in _hash_values(truth_entry, f)}
     if not ours:
         return False
-    return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(platform_entry, f)})
+    theirs = comparable(platform_entry)
+    return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(theirs, f)})
 
 
 def _contradicts(truth_entry: dict, platform_entry: dict) -> bool:
     """A hash both entries declare, without a value in common."""
+    platform_entry = comparable(platform_entry)
     return any(
         (ours := set(_hash_values(truth_entry, f)))
         and (theirs := set(_hash_values(platform_entry, f)))
@@ -445,6 +472,7 @@ def build_native_model(
     systems: dict[str, NativeSystem] = {}
 
     scraped_systems = (scraped or {}).get("systems", {})
+    composite_zips = (scraped or {}).get("zip_digest") == "composite"
 
     # Pass 1: the platform's own file, grouped as the platform groups it.
     for sys_key, sys_data in scraped_systems.items():
@@ -467,6 +495,7 @@ def build_native_model(
                     destination=destination,
                     native_system=native_id,
                     platform=file_entry,
+                    composite_md5=file_entry.get("zip_digest") == "composite",
                 )
             )
             report.files_kept += 1
@@ -545,7 +574,9 @@ def build_native_model(
                 checked = checked_hashes(truth_entry) or {}
                 for field_name in HASH_FIELDS:
                     ours = set(_hash_values(checked, field_name))
-                    theirs = set(_hash_values(matched.platform or {}, field_name))
+                    if field_name == "md5" and matched.composite_md5:
+                        continue
+                    theirs = set(_hash_values(comparable(matched.platform), field_name))
                     if ours and theirs and not ours & theirs:
                         matched.corrections.append(field_name)
                         report.hashes_corrected.append(
@@ -602,6 +633,7 @@ def build_native_model(
                     native_system=native_id,
                     truth=truth_entry,
                     foreign_system=native_id not in platform_ids,
+                    composite_md5=composite_zips and name.lower().endswith(".zip"),
                 )
             )
             report.files_added += 1

tests/test_exporters.py

@@ -43,6 +43,7 @@ from export_native import (  # noqa: E402
     SOURCES_INDEX,
     collect_originals,
     fetch,
+    landed_changes,
     pinned_base,
     source_key,
 )
@@ -1084,6 +1085,62 @@ class RecalboxExport(unittest.TestCase):
         }
         self.assertEqual(blocks, {pentagon: ["3" * 32], scorpion: [new]})
 
+    COMPOSITE_SCRAPED = {
+        "zip_digest": "composite",
+        "systems": {"snk-neogeo-cd": {"native_id": "neogeocd", "files": [{
+            "name": "neocdz.zip", "destination": "neocdz.zip", "md5": "a" * 32,
+            "native_system": "neogeocd", "core": "libretro/fbneo",
+            "required": True, "zip_digest": "composite",
+        }]}},
+    }
+
+    def test_a_composite_md5_is_not_replaced_by_a_container_md5(self):
+        """Recalbox hashes a .zip by its members (Bios.cpp:64-68,
+        Zip::Md5Composite): the truth's container md5 is another digest."""
+        truth = {"systems": {"snk-neogeo-cd": {"files": [
+            {"name": "neocdz.zip", "md5": "b" * 32, "required": True, "_cores": ["geolith"]},
+            {"name": "boot-data.zip", "md5": "c" * 32, "required": False, "_cores": ["geolith"]},
+        ]}}}
+        systems, report = build_native_model(truth, self.COMPOSITE_SCRAPED)
+        exporter = discover_exporters()["recalbox"]()
+        produced = exporter.render(systems, report, {}, None)["es_bios.xml"]
+        md5_of = {b.get("path"): b.get("md5") for b in ET.fromstring(produced).iter("bios")}
+        self.assertEqual(md5_of, {"neocdz.zip": "a" * 32})
+        self.assertEqual(landed_changes(exporter, systems).records, [])
+
+    def test_a_container_md5_never_pairs_with_a_composite(self):
+        """neocd.bin's md5 equals the composite of a neocdz.zip: by hash the
+        two would be read as one renamed file."""
+        truth = {"systems": {"snk-neogeo-cd": {"files": [
+            {"name": "neocd.bin", "md5": "a" * 32, "required": True, "_cores": ["neocd"]},
+        ]}}}
+        systems, _ = build_native_model(truth, self.COMPOSITE_SCRAPED)
+        declared = next(fe for s in systems.values() for fe in s.files if fe.platform)
+        self.assertIsNone(declared.truth)
+
+    def test_the_live_export_keeps_every_zip_md5(self):
+        """No element of es_bios.xml names a .zip under a new md5: none can
+        be stated as the composite Recalbox computes."""
+        path = REPO_ROOT / "exports" / "recalbox" / "es_bios.xml"
+        original = UPSTREAM_CACHE / "recalbox" / "es_bios.xml"
+        if not original.is_file():
+            self.skipTest("no cached original for recalbox")
+
+        def zips(text: str) -> dict[tuple[str, str], str]:
+            return {
+                (s.get("platform"), b.get("path")): ",".join(sorted(
+                    v.strip().lower() for v in b.get("md5", "").split(",")
+                ))
+                for s in ET.fromstring(text).iter("system")
+                for b in s.iter("bios")
+                if b.get("path", "").lower().endswith(".zip")
+            }
+
+        before = zips(original.read_text(encoding="utf-8"))
+        after = zips(path.read_text(encoding="utf-8"))
+        self.assertEqual({k: v for k, v in after.items() if k in before}, before)
+        self.assertEqual(set(after) - set(before), set())
+
     def test_a_truncated_export_is_reported(self):
         exporter, systems, _ = self._render()
         empty = {