fix: pair truth entries hash-consistent first

Each truth entry claimed its platform line in turn, so the first one to
arrive won. dosbox-x's system.rom (alias bios.rom, no path) took
Batocera's np2kai/BIOS.ROM by name before np2kai's own np2kai/bios.rom
was tried, and the export wrote an IBM PCjr md5 on it. A path also
outranked a contradicting hash: the truth's MSX.rom joins bluemsx's path
and fmsx's md5, which replaced the blueMSX dump on Recalbox's and
System.dat's Machines/Shared Roms/MSX.rom while fmsx's own line held it.

pair_entries runs each test over every entry before the next, tests that
contradict no hash first. export_native and diff_truth both read it.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 14:24:00 +02:00
1 parent 19974e34b8
commit 181ad0c630
4 files changed
+155 -45

No files matched your search

+68 -30
View File
@@ -14,6 +14,7 @@ grouping is rebuilt from the per-file native_system the scrapers record.
from __future__ import annotations
import sys
from collections.abc import Callable
from dataclasses import dataclass, field
from pathlib import Path
@@ -317,26 +318,64 @@ def by_hash(truth_entry: dict, platform_entry: dict) -> bool:
return bool(ours & {v for f in HASH_FIELDS for v in _hash_values(platform_entry, f)})
def pair_truth_entry(
truth_entry: dict, candidates: list[dict], claimed: set[int]
) -> int | None:
"""Which platform entry a truth entry describes, by index, or None.
def _contradicts(truth_entry: dict, platform_entry: dict) -> bool:
"""A hash both entries declare, without a value in common."""
return any(
(ours := set(_hash_values(truth_entry, f)))
and (theirs := set(_hash_values(platform_entry, f)))
and not ours & theirs
for f in HASH_FIELDS
)
The one rule export_native and diff_truth read. Each test is tried
across every candidate before the next: with three IPL.bin under one
system, separated only by their path, a first-match-wins scan attached
the truth to whichever came first. An entry never answers for two.
def _agreeing(test: Callable[[dict, dict], bool]) -> Callable[[dict, dict], bool]:
def agrees(truth_entry: dict, platform_entry: dict) -> bool:
return test(truth_entry, platform_entry) and not _contradicts(
truth_entry, platform_entry
)
return agrees
_PAIRING_TESTS = (
_agreeing(by_destination),
_agreeing(by_name),
by_hash,
by_destination,
by_name,
)
def pair_entries(truth_entries: list[dict], candidates: list[dict]) -> dict[int, int]:
"""Which platform entry each truth entry describes, as index -> index.
The one rule export_native and diff_truth read. Each test runs over
every truth entry and every candidate before the next one does:
claimed entry by entry, dosbox-x's system.rom took np2kai/BIOS.ROM by
its alias before np2kai's own np2kai/bios.rom was tried, and wrote an
IBM PCjr md5 on it. A path or a name that contradicts a hash comes
after the hash: the truth's MSX.rom carries bluemsx's path and fmsx's
md5, and fmsx's own line holds that md5. An entry never answers for two.
"""
for test in (by_destination, by_name, by_hash):
for index, candidate in enumerate(candidates):
if index not in claimed and test(truth_entry, candidate):
return index
return None
pairs: dict[int, int] = {}
claimed: set[int] = set()
for test in _PAIRING_TESTS:
for t_index, truth_entry in enumerate(truth_entries):
if t_index in pairs:
continue
for index, candidate in enumerate(candidates):
if index not in claimed and test(truth_entry, candidate):
pairs[t_index] = index
claimed.add(index)
break
return pairs
def platform_declares(truth_entry: dict, candidates: list[dict]) -> bool:
"""A file another truth entry already claimed is still the platform's
file, not one it lacks: scph101.bin, declared by two PSX cores."""
file, not one it lacks: scph101.bin, declared by two PSX cores. The
same content under another name is not: fuse opens fuse/128-1.rom by
name, and Recalbox's fuse/128p-1.rom holding that dump does not serve
it."""
return any(
by_destination(truth_entry, c) or by_name(truth_entry, c) for c in candidates
)
@@ -432,23 +471,22 @@ def build_native_model(
continue
target_ids = _target_native_ids(truth_sid)
truth_files = [
entry for entry in truth_files
if (name := str(entry.get("name", "")))
and not name.startswith("_") and not _is_placeholder(name)
]
candidates = [
candidate
for native_id in target_ids
for candidate in systems.get(native_id, NativeSystem(native_id)).files
if candidate.truth is None
]
pairs = pair_entries(truth_files, [c.platform or {} for c in candidates])
for truth_entry in truth_files:
name = str(truth_entry.get("name", ""))
if not name or name.startswith("_") or _is_placeholder(name):
continue
candidates = [
candidate
for native_id in target_ids
for candidate in systems.get(
native_id, NativeSystem(native_id)
).files
if candidate.truth is None
]
index = pair_truth_entry(
truth_entry, [c.platform or {} for c in candidates], set()
)
for t_index, truth_entry in enumerate(truth_files):
name = str(truth_entry["name"])
index = pairs.get(t_index)
matched = candidates[index] if index is not None else None
if matched is not None:
+9 -10
View File
@@ -618,13 +618,12 @@ def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None:
def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
"""Compare files between truth and scraped for a single system.
Entries are paired by pair_truth_entry, the rule export_native applies:
destination, then name or alias in the same directory at the same size,
then any hash in common. Two rules had diverged, and diff_truth flagged
a hash the export did not correct while the export corrected hashes the
diff never named. An entry never answers for two.
Entries are paired by pair_entries, the rule export_native applies. Two
rules had diverged, and diff_truth flagged a hash the export did not
correct while the export corrected hashes the diff never named. An entry
never answers for two.
"""
from exporter.baseline import _is_placeholder, pair_truth_entry, platform_declares
from exporter.baseline import _is_placeholder, pair_entries, platform_declares
truth_files = [
fe for fe in truth_sys.get("files", [])
@@ -639,9 +638,10 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
extra_phantom: list[dict] = []
extra_unprofiled: list[dict] = []
claimed: set[int] = set()
for t_entry in truth_files:
s_position = pair_truth_entry(t_entry, scraped_files, claimed)
pairs = pair_entries(truth_files, scraped_files)
claimed = set(pairs.values())
for t_index, t_entry in enumerate(truth_files):
s_position = pairs.get(t_index)
if s_position is None:
if not platform_declares(t_entry, scraped_files):
missing.append(
@@ -652,7 +652,6 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
}
)
continue
claimed.add(s_position)
s_entry = scraped_files[s_position]
mismatch = _hash_mismatch(t_entry, s_entry)
if mismatch: