chore: merge audit fixes (export)

# Conflicts:
#	exports/recalbox.json
#	exports/retropie.json

scripts/export_native.py

@@ -509,7 +509,7 @@ def _added_fields(exporter: object, entry: object) -> dict:
     if "size" in carried and entry.size() is not None:
         written["size"] = entry.size()
     if "required" in carried:
-        written["required"] = entry.required
+        written["required"] = exporter.required_value(entry)
     return written
 
 
@@ -535,20 +535,25 @@ def landed_changes(exporter: object, systems: dict) -> LandedChanges:
                         "added", entry, None, None, _added_fields(exporter, entry)
                     ))
                 continue
+            # Asked of the exporter: a format that names each entry's cores
+            # reads every entry of those cores, not only the paired one.
+            if (
+                entry.platform.get("required") is not None
+                and exporter.states(entry, "required")
+                and exporter.required_value(entry) != bool(entry.platform["required"])
+            ):
+                records.append(_change(
+                    "requirement_corrected", entry, "required",
+                    bool(entry.platform["required"]), exporter.required_value(entry),
+                ))
             for field_name in entry.corrections:
-                if not exporter.states(entry, field_name):
+                if field_name == "required" or not exporter.states(entry, field_name):
                     continue
-                if field_name == "required":
-                    records.append(_change(
-                        "requirement_corrected", entry, "required",
-                        bool(entry.platform.get("required")), entry.required,
-                    ))
-                else:
-                    records.append(_change(
-                        "hash_corrected", entry, field_name,
-                        _platform_value(entry, field_name),
-                        exporter.written(entry, field_name),
-                    ))
+                records.append(_change(
+                    "hash_corrected", entry, field_name,
+                    _platform_value(entry, field_name),
+                    exporter.written(entry, field_name),
+                ))
             records.extend(
                 _change("hash_filled", entry, field_name, None,
                         exporter.written(entry, field_name))

scripts/exporter/base_exporter.py

@@ -200,6 +200,14 @@ class BaseExporter(ABC):
             return self.single_value(fe, field_name)
         return ",".join(fe.hashes(field_name))
 
+    def required_value(self, fe: NativeFile) -> bool:
+        """The requirement the render writes for an entry.
+
+        The model's flag is true when any core needs the file. A format
+        that names the cores of each entry narrows it to those cores.
+        """
+        return fe.required
+
     def states(self, fe: NativeFile, field_name: str) -> bool:
         """Whether the written file carries this entry's corrected field.
 

scripts/exporter/baseline.py

@@ -61,6 +61,20 @@ def checked_hashes(entry: dict | None) -> dict | None:
     return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds}
 
 
+def comparable(entry: dict | None) -> dict:
+    """The entry with only the hashes the truth can be compared with.
+
+    Recalbox hashes a .zip by its members (Bios.cpp:64-68 calls
+    Zip::Md5Composite, which feeds each member's bytes in name order into
+    one md5), never by the container's bytes. Its md5 on an archive is that
+    composite, a digest no profile declares: read as a container md5 it was
+    replaced by geolith's, which Recalbox never computes.
+    """
+    if not entry or entry.get("zip_digest") != "composite":
+        return entry or {}
+    return {k: v for k, v in entry.items() if k != "md5"}
+
+
 def is_label(entry: dict) -> bool:
     """A name only content-scanning cores give: no core opens that file.
 
@@ -104,6 +118,14 @@ class NativeFile:
     # For an addition, each core that wants it as (runs standalone, name the
     # platform gives it), empty when no core the platform runs is named.
     runners: list[tuple[bool, str]] = field(default_factory=list)
+    # The platform reads this file's md5 as a composite of its members
+    # (Recalbox on a .zip): the truth's md5 describes the container and
+    # can be neither compared with it nor written in its place.
+    composite_md5: bool = False
+    # Every truth entry of the system that answers to this platform entry's
+    # name, whichever one the pairing took: the element names its cores, and
+    # each core's own entry for the file says whether that core needs it.
+    namesakes: list[dict] = field(default_factory=list)
 
     @property
     def origin(self) -> str:
@@ -129,7 +151,9 @@ class NativeFile:
         truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and
         reading it as a contradiction emptied the md5 the frontend checks.
         """
-        truth, plat = _field_sets(self._checked_truth), _field_sets(self.platform)
+        truth, plat = _field_sets(self._checked_truth), _field_sets(comparable(self.platform))
+        if self.composite_md5:
+            truth["md5"] = set()
         if not any(truth.values()):
             return "platform"
         if not any(plat.values()):
@@ -152,6 +176,8 @@ class NativeFile:
         the divergence is recorded, never silently merged: an emulator that
         rejects a file will reject it whatever the platform declares.
         """
+        if field_name == "md5" and self.composite_md5:
+            return _hash_values(self.platform or {}, field_name)
         truth_values = _hash_values(self._checked_truth, field_name)
         platform_values = _hash_values(self.platform or {}, field_name)
         source = self._identity_source()
@@ -186,6 +212,9 @@ class NativeFile:
         chosen among those the truth accepts. A field only the truth names
         is written when it names one value, which is then that dump's.
         """
+        if field_name == "md5" and self.composite_md5:
+            values = _hash_values(self.platform or {}, field_name)
+            return values[0] if values else ""
         source = self._identity_source()
         truth_values = _hash_values(self._checked_truth, field_name)
         platform_values = _hash_values(self.platform or {}, field_name)
@@ -365,11 +394,13 @@ def by_hash(truth_entry: dict, platform_entry: dict) -> bool:
     ours = {v for f in HASH_FIELDS for v in _hash_values(truth_entry, f)}
     if not ours:
         return False
-    return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(platform_entry, f)})
+    theirs = comparable(platform_entry)
+    return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(theirs, f)})
 
 
 def _contradicts(truth_entry: dict, platform_entry: dict) -> bool:
     """A hash both entries declare, without a value in common."""
+    platform_entry = comparable(platform_entry)
     return any(
         (ours := set(_hash_values(truth_entry, f)))
         and (theirs := set(_hash_values(platform_entry, f)))
@@ -445,6 +476,7 @@ def build_native_model(
     systems: dict[str, NativeSystem] = {}
 
     scraped_systems = (scraped or {}).get("systems", {})
+    composite_zips = (scraped or {}).get("zip_digest") == "composite"
 
     # Pass 1: the platform's own file, grouped as the platform groups it.
     for sys_key, sys_data in scraped_systems.items():
@@ -467,6 +499,7 @@ def build_native_model(
                     destination=destination,
                     native_system=native_id,
                     platform=file_entry,
+                    composite_md5=file_entry.get("zip_digest") == "composite",
                 )
             )
             report.files_kept += 1
@@ -534,6 +567,16 @@ def build_native_model(
             if candidate.truth is None
         ]
         pairs = pair_entries(truth_files, [c.platform or {} for c in candidates])
+        for candidate in (
+            c
+            for native_id in target_ids
+            for c in systems.get(native_id, NativeSystem(native_id)).files
+            if c.platform is not None
+        ):
+            own = str(candidate.name).casefold()
+            candidate.namesakes.extend(
+                entry for entry in truth_files if own in _names(entry)
+            )
 
         for t_index, truth_entry in enumerate(truth_files):
             name = str(truth_entry["name"])
@@ -545,7 +588,9 @@ def build_native_model(
                 checked = checked_hashes(truth_entry) or {}
                 for field_name in HASH_FIELDS:
                     ours = set(_hash_values(checked, field_name))
-                    theirs = set(_hash_values(matched.platform or {}, field_name))
+                    if field_name == "md5" and matched.composite_md5:
+                        continue
+                    theirs = set(_hash_values(comparable(matched.platform), field_name))
                     if ours and theirs and not ours & theirs:
                         matched.corrections.append(field_name)
                         report.hashes_corrected.append(
@@ -602,6 +647,7 @@ def build_native_model(
                     native_system=native_id,
                     truth=truth_entry,
                     foreign_system=native_id not in platform_ids,
+                    composite_md5=composite_zips and name.lower().endswith(".zip"),
                 )
             )
             report.files_added += 1

scripts/exporter/misterfpga_exporter.py

@@ -113,6 +113,7 @@ class Exporter(BaseExporter):
         database = json.loads(original, object_pairs_hook=OrderedDict)
         indexed = self._by_path(systems)
         self._written = set()
+        self._unsized: list[str] = []
 
         for path, entry in database.get("files", {}).items():
             fe = indexed.get(path)
@@ -124,9 +125,17 @@ class Exporter(BaseExporter):
                 # entry without one, a new size beside the old hash describes
                 # no file, and the entry is left as MiSTer wrote it.
                 continue
+            size = fe.size()
+            if md5 != entry.get("hash") and entry.get("size") and not size:
+                # Downloader refuses an entry without an integer size
+                # (db_entity.py:187) and compares it after each fetch
+                # (fetch_file_worker.py:127). The old size beside the new
+                # hash describes the dump the truth rejects, so the entry
+                # stays as MiSTer wrote it.
+                self._unsized.append(path)
+                continue
             self._written.add(id(fe))
             entry["hash"] = md5
-            size = fe.size()
             if size:
                 entry["size"] = size
 
@@ -143,6 +152,14 @@ class Exporter(BaseExporter):
             produced[_ARCHIVE] = archive if unchanged else rebuild_archive(archive, _DB_NAME, text)
         return produced
 
+    def withheld(self) -> list[str]:
+        """Each hash correction left out for want of the new dump's size."""
+        return [
+            f"{path} keeps its hash: the truth states no size for the new "
+            "dump, and the Downloader checks every entry by size"
+            for path in getattr(self, "_unsized", [])
+        ]
+
     def validate(
         self,
         systems: dict[str, NativeSystem],
@@ -169,6 +186,9 @@ class Exporter(BaseExporter):
         for path, fe in indexed.items():
             declared = files.get(path)
             md5 = fe.hash("md5")
-            if declared is not None and md5 and declared.get("hash") != md5:
+            if (
+                declared is not None and md5 and declared.get("hash") != md5
+                and path not in getattr(self, "_unsized", [])
+            ):
                 issues.append(f"hash not applied: {path}")
         return issues

scripts/exporter/recalbox_exporter.py

@@ -20,15 +20,17 @@ from xml.sax.saxutils import quoteattr
 
 sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
-from common import parse_untrusted_xml
+from common import load_emulator_profiles, parse_untrusted_xml
 
 from .base_exporter import BaseExporter
-from .baseline import NativeFile, NativeSystem, Report
+from .baseline import NativeFile, NativeSystem, Report, by_destination, by_name
 
 SOURCE_URL = (
     "https://gitlab.com/recalbox/recalbox/-/raw/master/board/recalbox/fsoverlay"
     "/recalbox/share_init/system/.emulationstation/es_bios.xml"
 )
+# Resolved from the file: the profiles say which names each core answers to.
+EMULATORS_DIR = Path(__file__).resolve().parents[2] / "emulators"
 SCHEMA_URL = (
     "https://gitlab.com/recalbox/recalbox/-/raw/master/board/recalbox/fsoverlay"
     "/recalbox/share_init/system/.emulationstation/es_bios.xsd"
@@ -149,12 +151,82 @@ class Exporter(BaseExporter):
             return path
         return fe.destination or fe.name
 
+    def _profile_keys(self) -> dict[str, str]:
+        """Each name a core answers to, as Recalbox writes it, to its profile."""
+        index = getattr(self, "_keys", None)
+        if index is None:
+            index = {}
+            for key, profile in load_emulator_profiles(str(EMULATORS_DIR)).items():
+                for name in (key, *(str(c) for c in profile.get("cores") or [])):
+                    index.setdefault(name.replace("-", "_").lower(), key)
+            self._keys = index
+        return index
+
+    def required_value(self, fe: NativeFile) -> bool:
+        """Whether the cores the element names need the file.
+
+        The element lists its cores, and each core's own entry for the file,
+        found by name, says whether that core needs it. The md5 pairing
+        takes one entry only: saturn/saturn_bios.bin names yabause,
+        yabasanshiro and kronos, which require it, and its md5 paired it
+        with the beetle_saturn entry. The merged flag is true when any core
+        needs the file: cdi/cdibios.zip names same_cdi, which declares no
+        such set, and became mandatory because MAME requires it. Where none
+        of the element's cores declares the file, the platform's value
+        stands.
+        """
+        entries = [fe.truth] if fe.truth else []
+        entries += [
+            e for e in fe.namesakes
+            if e is not fe.truth and self._same_file(e, fe.platform or {})
+        ]
+        if not entries:
+            return fe.required
+        index = self._profile_keys()
+        named = {
+            index.get(core.rsplit("/", 1)[-1].replace("-", "_").lower(), "")
+            for core in fe.cores()
+        } - {""}
+        readers: set[str] = set()
+        requirers: set[str] = set()
+        for entry in entries:
+            cores = set(entry.get("_cores") or ()) & named
+            readers |= cores
+            required_by = entry.get("_required_by")
+            if required_by is None:
+                if entry.get("required") is not False:
+                    requirers |= cores
+            else:
+                requirers |= cores & set(required_by)
+        if not readers:
+            if fe.platform is not None and fe.platform.get("required") is not None:
+                return bool(fe.platform["required"])
+            return fe.required
+        return bool(requirers)
+
+    @staticmethod
+    def _same_file(entry: dict, declared: dict) -> bool:
+        """Whether a truth entry of that name describes this element's file.
+
+        Its path or its name in the same directory does. A name under an
+        unrelated directory does too: kronos reads kronos/saturn_bios.bin,
+        and Recalbox files the same BIOS under saturn/. A name in another
+        directory of the same tree does not: bluemsx reads
+        Machines/Shared Roms/KANJI.rom, which is not Recalbox's
+        Machines/MSX2+ - European/KANJI.rom.
+        """
+        if by_destination(entry, declared) or by_name(entry, declared):
+            return True
+        theirs = str(declared.get("destination") or declared.get("name", "")).split("/")[:-1]
+        ours = str(entry.get("path") or entry.get("name", "")).split("/")[:-1]
+        return not {d.casefold() for d in theirs} & {d.casefold() for d in ours}
+
     def _bios_element(self, fe: NativeFile, native_id: str) -> str:
         attrs = [f"path={quoteattr(self._path(fe))}"]
         attrs.append(f'md5={quoteattr(",".join(fe.hashes("md5")))}')
         attrs.append(f'core={quoteattr(",".join(fe.cores()))}')
 
-        if not fe.required:
+        if not self.required_value(fe):
             attrs.append('mandatory="false"')
         elif fe.native("mandatory_declared", None) is True:
             attrs.append('mandatory="true"')
@@ -184,6 +256,19 @@ class Exporter(BaseExporter):
             return False
         return super().states(fe, field_name)
 
+    def lands(self, fe: NativeFile) -> bool:
+        """An element of theirs the render cannot rewrite is still in the
+        file, written back as Recalbox wrote it."""
+        return self.writable(fe) or id(fe) in getattr(self, "_held_whole", {})
+
+    def withheld(self) -> list[str]:
+        """Each element kept as written because the truth names no md5."""
+        return [
+            f"{label} keeps its md5: the truth accepts another dump and "
+            "names no md5 for it, which es_bios.xsd requires"
+            for label in sorted(getattr(self, "_held_whole", {}).values())
+        ]
+
     @classmethod
     def writable(cls, fe: NativeFile, require: str = "") -> bool:
         """es_bios.xsd makes md5 and core required: without them, no element.
@@ -234,6 +319,21 @@ class Exporter(BaseExporter):
                     for s in root.iter("system")
                 ]
 
+        # Elements of theirs written back as they stand, by (platform, path).
+        held = {
+            (platform, " ".join(child.get("path", "").split()))
+            for platform, _, children in theirs
+            for child in children
+            if isinstance(child, dict)
+        }
+        self._held_whole = {
+            id(fe): f"{system.native_id}/{self._path(fe)}"
+            for system in systems.values()
+            for fe in system.files
+            if fe.platform is not None
+            and not self.writable(fe)
+            and (system.native_id, " ".join(self._path(fe).split())) in held
+        }
         ours: dict[str, tuple[NativeSystem, dict[str, tuple[NativeFile, str]]]] = {}
         for system in systems.values():
             writable = [fe for fe in system.files if self.writable(fe)]

scripts/exporter/retropie_exporter.py

@@ -316,6 +316,7 @@ class Exporter(BaseExporter):
                     str(a).lower() for a in (fe.native("aliases", []) or [])
                 )
         self._skipped: dict[str, list[str]] = {}
+        self._named: set[int] = set()
         produced: dict[str, str] = {}
 
         def skip(reason: str, module: str) -> None:
@@ -360,7 +361,8 @@ class Exporter(BaseExporter):
                     | {str(a).lower() for a in (fe.native("aliases", []) or [])}) & listed
                 and (slot := self._slot_of(fe)) is not None
             }
-            missing = self._in_search_order(self._one_per_slot(candidates, filled))
+            proposed = self._one_per_slot(candidates, filled)
+            missing = self._in_search_order(proposed)
             if not missing:
                 continue
 
@@ -383,9 +385,21 @@ class Exporter(BaseExporter):
                 f'rp_module_help="{new_help}"',
                 1,
             )
+            self._named.update(id(fe) for fe in proposed)
 
         return produced
 
+    def lands(self, fe: NativeFile) -> bool:
+        """An addition lands only where a help sentence now names it.
+
+        Every file a profiled core reads is writable in principle, yet the
+        render appends names to a handful of sentences: counting the rest
+        listed 1360 additions for one changed scriptmodule.
+        """
+        if fe.platform is not None:
+            return self.writable(fe)
+        return id(fe) in getattr(self, "_named", set())
+
     def outcome(
         self,
         systems: dict[str, NativeSystem],

scripts/exporter/romm_exporter.py

@@ -26,6 +26,10 @@ SOURCE_URL = (
 )
 
 
+# Fixture field, model field, in the order RomM writes them.
+_FIELDS = (("size", "size"), ("crc", "crc32"), ("md5", "md5"), ("sha1", "sha1"))
+
+
 class Exporter(BaseExporter):
     """Write RomM's known_bios_files.json, corrected."""
 
@@ -35,6 +39,8 @@ class Exporter(BaseExporter):
         # addition that claims each other key first: both set by render.
         self._platform_keys: set[tuple[str, str]] = set()
         self._first_addition: dict[tuple[str, str], int] = {}
+        # Entries of theirs the truth takes over without a size: left whole.
+        self._unsized: dict[int, str] = {}
 
     @staticmethod
     def platform_name() -> str:
@@ -112,6 +118,7 @@ class Exporter(BaseExporter):
             if fe.platform is not None
         }
         self._first_addition = {}
+        self._unsized = {}
         self.normalized: list[str] = []
         self.normalized_values: list[tuple[NativeFile, str, str, str]] = []
         for system in sorted(systems.values(), key=lambda s: s.native_id):
@@ -133,11 +140,22 @@ class Exporter(BaseExporter):
                 # Their entry keeps its fields and their order, ours fill in.
                 entry: OrderedDict[str, str] = OrderedDict(output.get(key, {}))
                 # The fixture states every value as a string, size included.
-                # Where the model settles no size, RomM's own value stands.
                 # str(None) wrote "None", and RomM's int() on it raised.
                 size = fe.size()
-                if size is None and fe.platform is not None:
-                    size = fe.platform.get("size")
+                if size is None and fe.platform is not None and fe.platform.get("size"):
+                    # The truth took the entry over without a size: RomM's
+                    # size describes the dump the truth rejects, and RomM
+                    # compares the size before any hash, so the new hashes
+                    # beside it never verify. The entry stays as written.
+                    self._unsized[id(fe)] = key
+                    if key not in output:
+                        output[key] = OrderedDict(
+                            (field, value)
+                            for field, name in _FIELDS
+                            if (value := ",".join(_hash_values(fe.platform, name))
+                                if name != "size" else str(fe.platform["size"]))
+                        )
+                    continue
                 if size is not None:
                     self._write(entry, "size", str(int(size)), fe, "size")
                 for field, hash_name in (("crc", "crc32"), ("md5", "md5"), ("sha1", "sha1")):
@@ -160,6 +178,20 @@ class Exporter(BaseExporter):
         does not keep."""
         return bool(_hash_values(fe.platform or {}, name)) and not fe.hashes(name)
 
+    def states(self, fe: NativeFile, field_name: str) -> bool:
+        """An entry left whole carries none of the truth's values."""
+        if id(fe) in self._unsized:
+            return False
+        return super().states(fe, field_name)
+
+    def withheld(self) -> list[str]:
+        """Each entry kept whole for want of the new dump's size."""
+        return [
+            f"{key} keeps its values: the truth states no size for the new "
+            "dump, and RomM compares the size before any hash"
+            for key in sorted(self._unsized.values())
+        ]
+
     def written(self, fe: NativeFile, field_name: str) -> str:
         """A record holds one value per field, the one record_hash names."""
         return fe.record_hash(field_name)