chore: merge audit batch, truth and export

scripts/exporter/baseline.py

@@ -84,6 +84,26 @@ def comparable(entry: dict | None) -> dict:
     return {k: v for k, v in entry.items() if k != "md5"}
 
 
+def contradicted_hashes(truth_entry: dict, platform_entry: dict | None) -> list[str]:
+    """The hash fields a platform entry declares against the truth.
+
+    A field contradicts when both sides declare it without a value in
+    common, among the hashes the code checks and the platform values a
+    profile can be compared with. export_native corrects exactly these and
+    diff_truth reports them: the diff read three fields and let one shared
+    value settle all of them, so it stayed silent on an md5 or a sha256
+    the export rewrote.
+    """
+    checked = checked_hashes(truth_entry) or {}
+    theirs = comparable(platform_entry)
+    return [
+        field_name for field_name in HASH_FIELDS
+        if (ours := set(_hash_values(checked, field_name)))
+        and (held := set(_hash_values(theirs, field_name)))
+        and not ours & held
+    ]
+
+
 def is_label(entry: dict) -> bool:
     """A name only content-scanning cores give: no core opens that file.
 
@@ -595,17 +615,18 @@ def build_native_model(
             if matched is not None:
                 matched.truth = truth_entry
                 checked = checked_hashes(truth_entry) or {}
+                contradicted = contradicted_hashes(truth_entry, matched.platform)
                 for field_name in HASH_FIELDS:
-                    ours = set(_hash_values(checked, field_name))
                     if field_name == "md5" and matched.composite_md5:
                         continue
-                    theirs = set(_hash_values(comparable(matched.platform), field_name))
-                    if ours and theirs and not ours & theirs:
+                    if field_name in contradicted:
                         matched.corrections.append(field_name)
                         report.hashes_corrected.append(
                             f"{matched.native_system}/{matched.name} {field_name}"
                         )
-                    elif ours and not theirs:
+                    elif _hash_values(checked, field_name) and not _hash_values(
+                        comparable(matched.platform), field_name
+                    ):
                         matched.filled.append(field_name)
                 t_req = truth_entry.get("required")
                 p_req = (matched.platform or {}).get("required")

scripts/exporter/misterfpga_exporter.py

@@ -86,8 +86,14 @@ class Exporter(BaseExporter):
         return super().states(fe, field_name)
 
     def written(self, fe: NativeFile, field_name: str) -> str:
-        """An entry is checked against one md5, the first accepted."""
-        return fe.hash(field_name)
+        """An entry is one record: its md5, size and url describe one dump.
+
+        The first value the truth accepts replaced MiSTer's own accepted
+        md5 while the url still served MiSTer's dump, which the Downloader
+        then refused (fetch_file_worker.py:127), and the report said
+        nothing. record_hash keeps the platform's value when it is accepted.
+        """
+        return fe.record_hash(field_name)
 
     def _by_path(self, systems: dict[str, NativeSystem]) -> dict[str, object]:
         indexed: dict[str, object] = {}
@@ -119,7 +125,7 @@ class Exporter(BaseExporter):
             fe = indexed.get(path)
             if fe is None:
                 continue
-            md5 = fe.hash("md5")
+            md5 = self.written(fe, "md5")
             if not md5:
                 # The record is checked by its md5: where the truth takes the
                 # entry without one, a new size beside the old hash describes
@@ -185,7 +191,7 @@ class Exporter(BaseExporter):
         indexed = self._by_path(systems)
         for path, fe in indexed.items():
             declared = files.get(path)
-            md5 = fe.hash("md5")
+            md5 = self.written(fe, "md5")
             if (
                 declared is not None and md5 and declared.get("hash") != md5
                 and path not in getattr(self, "_unsized", [])

scripts/exporter/retropie_exporter.py

@@ -156,6 +156,30 @@ class Exporter(BaseExporter):
 
     @classmethod
     def _enumeration(cls, help_text: str) -> tuple[int, int, list[str]] | None:
+        """The first BIOS clause's list of names to extend, or None."""
+        clauses = cls._clauses(help_text)
+        return clauses[0][2] if clauses else None
+
+    @classmethod
+    def _clauses(
+        cls, help_text: str
+    ) -> list[tuple[int, list[str], tuple[int, int, list[str]] | None]]:
+        """Each BIOS clause that names files: its start, its names, and the
+        list to extend or None when it cannot be extended.
+
+        flycast's help has a Dreamcast clause and a Naomi/Atomiswave one.
+        Only the first was read, and naomi2.zip and segasp.zip were added to
+        the Dreamcast list.
+        """
+        found = []
+        for clause in _BIOS_CLAUSE.finditer(help_text):
+            names = [m.group(0) for m in cls._names_in(clause.group(0))]
+            if names:
+                found.append((clause.start(), names, cls._extendable(clause)))
+        return found
+
+    @classmethod
+    def _extendable(cls, clause: re.Match[str]) -> tuple[int, int, list[str]] | None:
         """The list of names to extend: its span and its names, or None.
 
         A list of alternatives ("a or b", "a/b") is not an enumeration: an
@@ -164,9 +188,6 @@ class Exporter(BaseExporter):
         are a list of examples ("eg. a, b") and one with words between its
         names, which a rewrite would lose.
         """
-        clause = _BIOS_CLAUSE.search(help_text)
-        if clause is None:
-            return None
         text = clause.group(0)
         names = cls._names_in(text)
         if not names:
@@ -308,13 +329,12 @@ class Exporter(BaseExporter):
         by_core = self._files_by_core(systems)
         # Aliases count as names we know: RetroPie writes dc_flash.bin where
         # flycast's profile files it as an alias of another primary.
-        known: set[str] = set()
+        systems_of: dict[str, set[str]] = {}
         for system in systems.values():
             for fe in system.files:
-                known.add(fe.name.lower())
-                known.update(
-                    str(a).lower() for a in (fe.native("aliases", []) or [])
-                )
+                for name in [fe.name, *(fe.native("aliases", []) or [])]:
+                    systems_of.setdefault(str(name).lower(), set()).add(fe.native_system)
+        known = set(systems_of)
         self._skipped: dict[str, list[str]] = {}
         self._named: set[int] = set()
         produced: dict[str, str] = {}
@@ -362,30 +382,52 @@ class Exporter(BaseExporter):
                 and (slot := self._slot_of(fe)) is not None
             }
             proposed = self._one_per_slot(candidates, filled)
-            missing = self._in_search_order(proposed)
-            if not missing:
+            if not proposed:
                 continue
 
-            enumeration = self._enumeration(help_text)
-            if enumeration is None:
-                # No enumeration to extend, and a sentence we would have to
-                # write ourselves is a documentation change, not a correction.
-                skip("whose help text lists no files to add to", module_id)
+            # A file joins the list of its own system: the clause whose
+            # names the platform files under that system, and only one.
+            clauses = self._clauses(help_text)
+            homes: dict[int, list[NativeFile]] = {}
+            homeless = False
+            for fe in proposed:
+                owners = [
+                    i for i, (_, names, _) in enumerate(clauses)
+                    if fe.native_system in set().union(
+                        *(systems_of.get(n.lower(), set()) for n in names))
+                ]
+                if len(owners) == 1 and clauses[owners[0]][2] is not None:
+                    homes.setdefault(owners[0], []).append(fe)
+                else:
+                    homeless = True
+            if homeless:
+                # No enumeration of that system to extend, and a sentence we
+                # would have to write ourselves is a documentation change,
+                # not a correction.
+                skip("whose help text lists no files of that system to add to", module_id)
+            if not homes:
                 continue
 
-            if len(listed) + len(missing) > self._longest:
+            added = sum(len(self._in_search_order(fes)) for fes in homes.values())
+            if len(listed) + added > self._longest:
                 skip("whose help text would grow longer than any it has today", module_id)
                 continue
 
-            new_help = self._extended(help_text, enumeration, missing)
-            if len(listed) + len(missing) > 1:
-                new_help = _BIOS_NOUN.sub(r"\1 files", new_help, count=1)
+            new_help = help_text
+            for i in sorted(homes, reverse=True):
+                start, _, enumeration = clauses[i]
+                missing = self._in_search_order(homes[i])
+                new_help = self._extended(new_help, enumeration, missing)
+                if len(enumeration[2]) + len(missing) > 1:
+                    new_help = new_help[:start] + _BIOS_NOUN.sub(
+                        r"\1 files", new_help[start:enumeration[0]], count=1
+                    ) + new_help[enumeration[0]:]
+                self._named.update(id(fe) for fe in homes[i])
             produced[relative] = originals[relative].replace(
                 f'rp_module_help="{help_text}"',
                 f'rp_module_help="{new_help}"',
                 1,
             )
-            self._named.update(id(fe) for fe in proposed)
 
         return produced
 

scripts/truth.py

@@ -18,7 +18,7 @@ from common import (
     resolve_platform_cores,
     standalone_set as platform_standalone_set,
 )
-from hashing import compute_hashes, parse_md5_list
+from hashing import compute_hashes
 from validation import (
     filter_files_by_mode,
     installed_with_emulator,
@@ -113,7 +113,6 @@ _FILLED_FIELDS = (
     "note",
     "aliases",
     "contents",
-    "region",
 )
 _COPIED_FIELDS = (
     "sha1",
@@ -290,6 +289,27 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None:
         if file_entry.get(field) is not None and existing.get(field) is None:
             existing[field] = file_entry[field]
     _merge_priority(existing, file_entry.get("priority"))
+    _merge_region(existing, file_entry.get("region"))
+
+
+def _merge_region(existing: dict, theirs: str | list | None) -> None:
+    """The regions every reader selects the file for, the pack's own rule.
+
+    region.build_region_index takes the union of the tags and keeps a file
+    untagged as soon as one reader declares it without a region. The first
+    tag used to win: psxonpsp660.bin came out japan from pcsx_rearmed while
+    SwanStation tags it world and beetle_psx not at all.
+    """
+    if existing.get("_region_void"):
+        return
+    if not theirs:
+        existing.pop("region", None)
+        existing["_region_void"] = True
+        return
+    ours = existing.get("region") or []
+    ours = ours if isinstance(ours, list) else [ours]
+    added = [r for r in (theirs if isinstance(theirs, list) else [theirs]) if r not in ours]
+    existing["region"] = [*ours, *added]
 
 
 def _merge_hashes(
@@ -373,6 +393,8 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
     crc = entry.get("crc32")
     if isinstance(crc, str):
         entry["crc32"] = crc.removeprefix("0x").lower()
+    if not file_entry.get("region"):
+        entry["_region_void"] = True
     entry["_cores"] = {emu_name}
     entry["_required_by"] = {emu_name} if file_entry.get("required") else set()
     entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set()
@@ -486,14 +508,19 @@ def _sizes_clash(a: dict, b: dict) -> bool:
     return False
 
 
-def _identity_values(entry: dict, db: dict | None) -> set[str]:
-    """Every hash an entry declares, with the siblings the database knows."""
+def _identity(entry: dict, db: dict | None) -> dict:
+    """The entry with the hashes and size the database knows for its hash."""
     probe = {k: entry[k] for k in ("sha1", "md5", "crc32", "size") if entry.get(k)}
     if db and probe and not any(isinstance(v, list) for v in probe.values()):
         _enrich_hashes(probe, db)
+    return {**entry, **{k: v for k, v in probe.items() if not entry.get(k)}}
+
+
+def _identity_values(entry: dict) -> set[str]:
+    """Every hash an identity declares."""
     values: set[str] = set()
     for field in ("sha1", "md5", "sha256", "crc32"):
-        values |= _declared(entry, field) | _declared(probe, field)
+        values |= _declared(entry, field)
     return values
 
 
@@ -503,10 +530,14 @@ def _declaration_verdict(entry: dict, declaration: dict, db: dict | None) -> str
     A shared hash proves it. Sizes no value of which agrees refute it, as
     name_match_size_ok refutes a candidate. Two hashes at one size are two
     revisions of one slot, as slots.py reads them, so they refute nothing.
+    The size the database holds for a declared hash counts as declared:
+    Batocera's BK BASIC10.ROM names only an md5, of a 24576-byte file, and
+    clk's 16 KB Oric basic10.rom was filed under the BK beside it.
     """
-    if _identity_values(entry, db) & _identity_values(declaration, db):
+    ours, theirs = _identity(entry, db), _identity(declaration, db)
+    if _identity_values(ours) & _identity_values(theirs):
         return "same"
-    if _sizes_clash(entry, declaration):
+    if _sizes_clash(ours, theirs):
         return "other"
     return "undecided"
 
@@ -814,6 +845,7 @@ def generate_platform_truth(
     for sys_id, sys_data in systems.items():
         for fe in sys_data.get("files", []):
             fe.pop("_hash_by", None)
+            fe.pop("_region_void", None)
             fe["_cores"] = sorted(fe.get("_cores", set()))
             fe["_required_by"] = sorted(fe.get("_required_by", set()))
             fe["_named_by"] = sorted(fe.get("_named_by", set()))
@@ -846,53 +878,30 @@ def generate_platform_truth(
 # Platform truth diffing
 
 
-def _hash_values(entry: dict, field: str) -> set[str]:
-    """The values a diffed entry declares for a hash field.
-
-    A profile writes a list, a platform YAML one comma-separated string
-    (Recalbox's multi-hash): both are read the way the exporter reads them.
-    A number is a hash YAML coerced and no longer spells: it is ignored.
-    """
-    raw = entry.get(field)
-    return set(parse_md5_list(raw)) if isinstance(raw, (str, list)) else set()
-
-
-def _hash_set(entry: dict) -> set[str]:
-    return {value for h in ("sha1", "md5", "crc32") for value in _hash_values(entry, h)}
-
-
 def _path_tail(value: object) -> str:
     return str(value or "").replace("\\", "/").casefold()
 
 
 def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None:
-    """The first hash both sides declare without a value in common.
+    """The first hash field the export would correct, by the export's rule.
 
-    A value shared on any field settles it: the truth holds every revision a
-    core accepts, and a profile may give one revision's sha1 without its md5.
-    A hash the code does not check contradicts nothing (checked_hashes), and
-    a platform md5 that is a composite of a zip's members is no container
-    md5 (comparable), the rule pair_entries applies.
+    contradicted_hashes is the one decision: a hash the code does not check
+    contradicts nothing (checked_hashes), and a platform md5 that is a
+    composite of a zip's members is no container md5 (comparable).
     """
-    from exporter.baseline import checked_hashes, comparable
+    from exporter.baseline import contradicted_hashes
 
-    s_entry = comparable(s_entry)
-    if _hash_set(t_entry) & _hash_set(s_entry):
+    fields = contradicted_hashes(t_entry, s_entry)
+    if not fields:
         return None
-    t_entry = checked_hashes(t_entry) or {}
-    for h in ("sha1", "md5", "crc32"):
-        t_hash = t_entry.get(h, "")
-        s_hash = s_entry.get(h, "")
-        t_values, s_values = _hash_values(t_entry, h), _hash_values(s_entry, h)
-        if t_values and s_values and not t_values & s_values:
-            return {
-                "name": s_entry["name"],
-                "hash_type": h,
-                f"truth_{h}": t_hash,
-                f"scraped_{h}": s_hash,
-                "truth_cores": list(t_entry.get("_cores", [])),
-            }
-    return None
+    h = fields[0]
+    return {
+        "name": s_entry["name"],
+        "hash_type": h,
+        f"truth_{h}": t_entry.get(h, ""),
+        f"scraped_{h}": s_entry.get(h, ""),
+        "truth_cores": list(t_entry.get("_cores", [])),
+    }
 
 
 def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:

scripts/validate_pr.py

@@ -34,6 +34,8 @@ from common import (
     parse_md5_list,
 )
 from hashing import parse_hash_list
+from nativemode import digest_algorithm
+from validation import frontend_digest_matches, inner_rom_check
 
 try:
     import yaml
@@ -140,7 +142,8 @@ def load_platform_hashes(platforms_dir: str, db: dict | None = None) -> dict:
     bios/<Manufacturer>/<Console>/ folders the collection files its system
     under, read from the other files of that system it holds.
     """
-    known = {"sha1": set(), "md5": set(), "names": set(), "folders": {}, "sizes": {}}
+    known = {"sha1": set(), "md5": set(), "sha256": set(), "crc32": set(),
+             "names": set(), "folders": {}, "sizes": {}, "entries": {}}
     systems_of: dict[str, set[str]] = {}
     folders_of: dict[str, dict[str, int]] = {}
     files = (db or {}).get("files", {})
@@ -160,15 +163,31 @@ def load_platform_hashes(platforms_dir: str, db: dict | None = None) -> dict:
             print(f"WARNING: {name}: {exc}", file=sys.stderr)
             continue
 
+        mode, rule = config.get("verification_mode"), config.get("hash_rule", "")
         for system_id, system in config.get("systems", {}).items():
             for file_entry in system.get("files", []):
                 sha1s = parse_hash_list(file_entry.get("sha1"))
                 md5s = parse_md5_list(file_entry.get("md5"))
+                sha256s = parse_hash_list(file_entry.get("sha256"))
                 known["sha1"].update(sha1s)
                 known["md5"].update(md5s)
+                known["sha256"].update(sha256s)
+                if not (sha1s or md5s or sha256s) and isinstance(file_entry.get("size"), int):
+                    # The resolver's reading: a crc32 counts where no stronger
+                    # hash is declared, bounded by the declared size.
+                    known["crc32"].update(
+                        (crc.zfill(8), file_entry["size"])
+                        for crc in parse_hash_list(file_entry.get("crc32"))
+                    )
                 if "name" in file_entry:
                     known["names"].add(file_entry["name"])
                     _note_sizes(known["sizes"], [file_entry["name"]], file_entry)
+                    # A frontend that hashes a ZIP's member or its members
+                    # together, or that takes any hash at the declared size,
+                    # answers by its own rule, read per entry.
+                    known["entries"].setdefault(file_entry["name"].lower(), []).append(
+                        (file_entry, mode, rule, name)
+                    )
                 for value in (*sha1s, *md5s):
                     systems_of.setdefault(value, set()).add(system_id)
                 _note_folder(folders_of, system_id, files, by_md5, sha1s, md5s)
@@ -354,6 +373,14 @@ def validate_file(
 
     sha1_known = result.sha1 in platform_hashes.get("sha1", set())
     md5_known = result.md5 in platform_hashes.get("md5", set())
+    other_known = (
+        hashes.get("sha256", "") in platform_hashes.get("sha256", set())
+        or (result.crc32, result.size) in platform_hashes.get("crc32", set())
+    )
+    accepted_by = (
+        set() if sha1_known or md5_known or other_known
+        else _platforms_accepting(filepath, result.filename, platform_hashes)
+    )
     name_known = result.filename in platform_hashes.get("names", set())
 
     profile_hashes = profile_hashes or {}
@@ -369,6 +396,12 @@ def validate_file(
         result.add_check(True, "SHA1 matches known platform requirement")
     elif md5_known:
         result.add_check(True, "MD5 matches known platform requirement")
+    elif other_known:
+        result.add_check(True, "Hash matches known platform requirement")
+    elif accepted_by:
+        result.add_check(
+            True, "Accepted by the platform's own check: " + ", ".join(sorted(accepted_by))
+        )
     elif by_profile:
         names = sorted(by_profile)
         shown = ", ".join(names[:5]) + (f" and {len(names) - 5} more" if len(names) > 5 else "")
@@ -415,10 +448,10 @@ def validate_file(
             "File is not under bios/ yet. Its system's files are in "
             + " or ".join(f"`{f}`" for f in expected_folders)
         )
-    elif _variant_home(db, result.filename):
+    elif _variant_home(db, result.filename, result.sha1):
         result.add_warning(
             "File is not under bios/ yet. The collection keeps this name in "
-            f"`{_variant_home(db, result.filename)}`"
+            f"`{_variant_home(db, result.filename, result.sha1)}`"
         )
     else:
         result.add_warning(
@@ -439,7 +472,8 @@ def validate_file(
             f"this file has {result.size:,}: truncated or another file"
         )
 
-    if name_known and not sha1_known and not md5_known and not by_profile and size_fits:
+    if (name_known and not (sha1_known or md5_known or other_known or accepted_by)
+            and not by_profile and size_fits):
         home = _variant_home(db, result.filename)
         result.add_info(
             "This may be a valid variant. If accepted, it will be placed in "
@@ -449,10 +483,43 @@ def validate_file(
     return result
 
 
-def _variant_home(db: dict | None, name: str) -> str:
-    """The folder the collection keeps a name in, for its variants beside it."""
+def _platforms_accepting(filepath: str, filename: str, platform_hashes: dict) -> set[str]:
+    """Platforms whose declaration of this name accepts the file by their
+    own check, as verify and the pack builder read it.
+
+    Batocera pins a ROM inside the ZIP (zipped_file), Recalbox hashes a
+    ZIP's members together, RomM takes any hash at the declared size: the
+    container's sha1 or md5 is in none of those lists, and adam_fdc.zip,
+    which Batocera accepts, was called an unknown variant.
+    """
+    accepted: set[str] = set()
+    for entry, mode, rule, platform in platform_hashes.get("entries", {}).get(
+        filename.lower(), []
+    ):
+        if platform in accepted:
+            continue
+        if entry.get("zipped_file"):
+            ok = inner_rom_check(entry, filepath) == "ok"
+        else:
+            algorithm = digest_algorithm(mode)
+            ok = bool(algorithm or rule) and frontend_digest_matches(
+                entry, filepath, algorithm or "", rule
+            )
+        if ok:
+            accepted.add(platform)
+    return accepted
+
+
+def _variant_home(db: dict | None, name: str, held: str = "") -> str:
+    """The folder the collection keeps a name in, for its variants beside it.
+
+    The folder of an identical copy comes first: adam_fdc.zip, held in
+    bios/Coleco/Adam/, was sent to bios/Arcade/MAME/, the first folder
+    the name index gave.
+    """
     files = (db or {}).get("files", {})
-    for sha1 in (db or {}).get("indexes", {}).get("by_name", {}).get(name, []):
+    by_name = (db or {}).get("indexes", {}).get("by_name", {}).get(name, [])
+    for sha1 in ([held] if held in files else []) + list(by_name):
         path = str(files.get(sha1, {}).get("path", ""))
         folder = path.rsplit("/", 1)[0] + "/" if "/" in path else ""
         if folder and "/.variants" not in "/" + folder: