fix: correct no hash the code does not check

validation: names what the code verifies. dosbox checks
MT32_CONTROL.ROM against nine sha1 and size; the truth's one md5 and
crc32 describe one of those revisions, and the export wrote them beside
System.dat's own sha1, a rom line no file satisfies, and replaced
Batocera's md5. pcsx_rearmed and DuckStation check scph1001.bin by size,
and the v2.0 md5 platforms name was replaced by v4.1's.

checked_hashes keeps a hash the code does not name only where a checked
hash pins one content. Otherwise it still pairs a renamed file but never
corrects or fills a value the platform wrote. Additions keep every hash.
diff_truth reads the same rule.
This commit is contained in:
Abdessamad Derraz committed 2026-10-20 14:40:10 +02:00
1 parent cc31f9f863
commit e81c63804a
3 files changed
+112 -4

No files matched your search

+39 -4
View File
@@ -35,6 +35,32 @@ def _hash_values(entry: dict, field_name: str) -> list[str]:
return [v.strip().lower() for v in str(raw).split(",") if v.strip()]
def checked_hashes(entry: dict | None) -> dict | None:
"""The truth entry with only the hashes its code checks.
`validation:` names what the code verifies. Where a checked hash names
one value, the content is pinned and every hash describes it. Where
the code checks no hash, or accepts several values, a hash it does not
name describes one revision among those it accepts: dosbox checks
MT32_CONTROL.ROM against nine sha1, and the one md5 beside them
replaced System.dat's, another accepted revision. Such a value still
pairs a renamed file, it never corrects or fills one.
"""
if not entry or entry.get("validation") is None:
return entry
raw = entry["validation"]
if isinstance(raw, dict):
kinds = {str(k) for values in raw.values() for k in values or ()}
elif isinstance(raw, list):
kinds = {str(k) for k in raw}
else:
kinds = {str(raw)}
checked = [f for f in HASH_FIELDS if f in kinds and _hash_values(entry, f)]
if checked and all(len(set(_hash_values(entry, f))) == 1 for f in checked):
return entry
return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds}
def _is_placeholder(name: str) -> bool:
return "<" in name or ">" in name or "*" in name
@@ -76,6 +102,14 @@ class NativeFile:
return "both"
return "platform" if self.platform is not None else "truth"
@property
def _checked_truth(self) -> dict:
"""The truth's hashes that can overrule the platform's. An addition
has no platform value to overrule and keeps them all."""
if self.platform is None:
return self.truth or {}
return checked_hashes(self.truth) or {}
def _identity_source(self) -> str:
"""Which side's content fields describe the file: one side, or both.
@@ -86,7 +120,7 @@ class NativeFile:
truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and
reading it as a contradiction emptied the md5 the frontend checks.
"""
truth, plat = _field_sets(self.truth), _field_sets(self.platform)
truth, plat = _field_sets(self._checked_truth), _field_sets(self.platform)
if not any(truth.values()):
return "platform"
if not any(plat.values()):
@@ -109,7 +143,7 @@ class NativeFile:
the divergence is recorded, never silently merged: an emulator that
rejects a file will reject it whatever the platform declares.
"""
truth_values = _hash_values(self.truth or {}, field_name)
truth_values = _hash_values(self._checked_truth, field_name)
platform_values = _hash_values(self.platform or {}, field_name)
source = self._identity_source()
if source == "platform":
@@ -139,7 +173,7 @@ class NativeFile:
is written when it names one value, which is then that dump's.
"""
source = self._identity_source()
truth_values = _hash_values(self.truth or {}, field_name)
truth_values = _hash_values(self._checked_truth, field_name)
platform_values = _hash_values(self.platform or {}, field_name)
if source == "truth":
return truth_values[0] if truth_values else ""
@@ -491,8 +525,9 @@ def build_native_model(
if matched is not None:
matched.truth = truth_entry
checked = checked_hashes(truth_entry) or {}
for field_name in HASH_FIELDS:
ours = set(_hash_values(truth_entry, field_name))
ours = set(_hash_values(checked, field_name))
theirs = set(_hash_values(matched.platform or {}, field_name))
if ours and theirs and not ours & theirs:
matched.corrections.append(field_name)
+4
View File
@@ -602,9 +602,13 @@ def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None:
A value shared on any field settles it: the truth holds every revision a
core accepts, and a profile may give one revision's sha1 without its md5.
A hash the code does not check contradicts nothing (checked_hashes).
"""
from exporter.baseline import checked_hashes
if _hash_set(t_entry) & _hash_set(s_entry):
return None
t_entry = checked_hashes(t_entry) or {}
for h in ("sha1", "md5", "crc32"):
t_hash = t_entry.get(h, "")
s_hash = s_entry.get(h, "")
+69
View File
@@ -258,6 +258,75 @@ class Reconciliation(unittest.TestCase):
self.assertEqual(added.destination, "fuse/128-1.rom")
self.assertEqual(report.files_added, 1)
def test_a_hash_the_code_does_not_check_corrects_nothing(self):
"""dosbox checks MT32_CONTROL.ROM by sha1 and size. The truth accepts
System.dat's sha1 but carries the md5 and crc of another revision,
and the export wrote those beside System.dat's sha1: a record no
file satisfies, counted as a correction."""
truth = {"systems": {"dos": {"files": [
{"name": "MT32_CONTROL.ROM", "sha1": ["1" * 40, "2" * 40],
"md5": "a" * 32, "crc32": "aaaaaaaa", "size": 65536,
"validation": ["sha1", "size"]},
]}}}
scraped = {"systems": {"dos": {"native_id": "dos", "files": [
{"name": "MT32_CONTROL.ROM", "destination": "MT32_CONTROL.ROM",
"native_system": "dos", "sha1": "2" * 40, "md5": "b" * 32,
"crc32": "bbbbbbbb", "size": 65536},
]}}}
systems, report = build_native_model(truth, scraped)
entry = systems["dos"].files[0]
self.assertEqual(
[entry.record_hash(f) for f in ("crc32", "md5", "sha1")],
["bbbbbbbb", "b" * 32, "2" * 40],
)
self.assertEqual(entry.hashes("md5"), ["b" * 32])
self.assertEqual(report.hashes_corrected, [])
def test_a_size_checked_file_keeps_the_dump_the_platform_names(self):
"""pcsx_rearmed and DuckStation check scph1001.bin by size. The
truth's md5 is v4.1's, Batocera names v2.0: the export replaced it
and the frontend then refused the v2.0 the cores load. Nor is a
blank md5 filled with one revision the code does not require."""
truth = {"systems": {"sony-playstation": {"files": [
{"name": "scph1001.bin", "md5": "a" * 32, "size": 524288,
"validation": ["size"]},
{"name": "scph5500.bin", "md5": "c" * 32, "size": 524288,
"validation": ["size"]},
]}}}
scraped = {"systems": {"sony-playstation": {"native_id": "psx", "files": [
{"name": "scph1001.bin", "destination": "scph1001.bin",
"native_system": "psx", "md5": "b" * 32},
{"name": "scph5500.bin", "destination": "scph5500.bin",
"native_system": "psx"},
]}}}
systems, report = build_native_model(truth, scraped)
v20, blank = systems["psx"].files
self.assertEqual(v20.hash("md5"), "b" * 32)
self.assertEqual(blank.hash("md5"), "")
self.assertEqual((report.hashes_corrected, blank.filled), ([], []))
def test_a_hash_beside_one_checked_value_still_corrects(self):
"""A core that checks one crc32 pins one dump, and the md5 of that
dump is no less true: a platform md5 against it is corrected."""
truth = {"systems": {"coco": {"files": [
{"name": "bas12.rom", "crc32": "aaaaaaaa", "md5": "a" * 32,
"validation": ["crc32"]},
]}}}
scraped = {"systems": {"coco": {"native_id": "coco", "files": [
{"name": "bas12.rom", "destination": "bas12.rom",
"native_system": "coco", "md5": "b" * 32},
]}}}
systems, report = build_native_model(truth, scraped)
self.assertEqual(systems["coco"].files[0].hash("md5"), "a" * 32)
self.assertEqual(report.hashes_corrected, ["coco/bas12.rom md5"])
def test_the_diff_names_no_mismatch_the_code_does_not_check(self):
from truth import _diff_system
truth_sys = {"files": [{"name": "scph1001.bin", "md5": "a" * 32,
"validation": ["size"]}]}
scraped_sys = {"files": [{"name": "scph1001.bin", "md5": "b" * 32}]}
self.assertNotIn("hash_mismatch", _diff_system(truth_sys, scraped_sys))
def test_the_diff_pairs_by_the_same_rule(self):
"""diff_truth reads the same pairing, or it names a mismatch the
export does not correct."""