fix: one hash mismatch rule for diff and export

diff_truth read sha1, md5 and crc32 and let one shared value settle the
entry, while export_native corrected each of the four fields on its own.
contradicted_hashes is now the single decision both read.
This commit is contained in:
Abdessamad Derraz committed 2026-11-05 11:50:09 +01:00
1 parent f69bd7cbd5
commit d7d6f3e412
3 files changed
+80 -45

No files matched your search

+25 -4
View File
@@ -84,6 +84,26 @@ def comparable(entry: dict | None) -> dict:
return {k: v for k, v in entry.items() if k != "md5"}
def contradicted_hashes(truth_entry: dict, platform_entry: dict | None) -> list[str]:
"""The hash fields a platform entry declares against the truth.
A field contradicts when both sides declare it without a value in
common, among the hashes the code checks and the platform values a
profile can be compared with. export_native corrects exactly these and
diff_truth reports them: the diff read three fields and let one shared
value settle all of them, so it stayed silent on an md5 or a sha256
the export rewrote.
"""
checked = checked_hashes(truth_entry) or {}
theirs = comparable(platform_entry)
return [
field_name for field_name in HASH_FIELDS
if (ours := set(_hash_values(checked, field_name)))
and (held := set(_hash_values(theirs, field_name)))
and not ours & held
]
def is_label(entry: dict) -> bool:
"""A name only content-scanning cores give: no core opens that file.
@@ -595,17 +615,18 @@ def build_native_model(
if matched is not None:
matched.truth = truth_entry
checked = checked_hashes(truth_entry) or {}
contradicted = contradicted_hashes(truth_entry, matched.platform)
for field_name in HASH_FIELDS:
ours = set(_hash_values(checked, field_name))
if field_name == "md5" and matched.composite_md5:
continue
theirs = set(_hash_values(comparable(matched.platform), field_name))
if ours and theirs and not ours & theirs:
if field_name in contradicted:
matched.corrections.append(field_name)
report.hashes_corrected.append(
f"{matched.native_system}/{matched.name} {field_name}"
)
elif ours and not theirs:
elif _hash_values(checked, field_name) and not _hash_values(
comparable(matched.platform), field_name
):
matched.filled.append(field_name)
t_req = truth_entry.get("required")
p_req = (matched.platform or {}).get("required")
+16 -39
View File
@@ -18,7 +18,7 @@ from common import (
resolve_platform_cores,
standalone_set as platform_standalone_set,
)
from hashing import compute_hashes, parse_md5_list
from hashing import compute_hashes
from validation import (
filter_files_by_mode,
installed_with_emulator,
@@ -878,53 +878,30 @@ def generate_platform_truth(
# Platform truth diffing
def _hash_values(entry: dict, field: str) -> set[str]:
"""The values a diffed entry declares for a hash field.
A profile writes a list, a platform YAML one comma-separated string
(Recalbox's multi-hash): both are read the way the exporter reads them.
A number is a hash YAML coerced and no longer spells: it is ignored.
"""
raw = entry.get(field)
return set(parse_md5_list(raw)) if isinstance(raw, (str, list)) else set()
def _hash_set(entry: dict) -> set[str]:
return {value for h in ("sha1", "md5", "crc32") for value in _hash_values(entry, h)}
def _path_tail(value: object) -> str:
return str(value or "").replace("\\", "/").casefold()
def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None:
"""The first hash both sides declare without a value in common.
"""The first hash field the export would correct, by the export's rule.
A value shared on any field settles it: the truth holds every revision a
core accepts, and a profile may give one revision's sha1 without its md5.
A hash the code does not check contradicts nothing (checked_hashes), and
a platform md5 that is a composite of a zip's members is no container
md5 (comparable), the rule pair_entries applies.
contradicted_hashes is the one decision: a hash the code does not check
contradicts nothing (checked_hashes), and a platform md5 that is a
composite of a zip's members is no container md5 (comparable).
"""
from exporter.baseline import checked_hashes, comparable
from exporter.baseline import contradicted_hashes
s_entry = comparable(s_entry)
if _hash_set(t_entry) & _hash_set(s_entry):
fields = contradicted_hashes(t_entry, s_entry)
if not fields:
return None
t_entry = checked_hashes(t_entry) or {}
for h in ("sha1", "md5", "crc32"):
t_hash = t_entry.get(h, "")
s_hash = s_entry.get(h, "")
t_values, s_values = _hash_values(t_entry, h), _hash_values(s_entry, h)
if t_values and s_values and not t_values & s_values:
return {
"name": s_entry["name"],
"hash_type": h,
f"truth_{h}": t_hash,
f"scraped_{h}": s_hash,
"truth_cores": list(t_entry.get("_cores", [])),
}
return None
h = fields[0]
return {
"name": s_entry["name"],
"hash_type": h,
f"truth_{h}": t_entry.get(h, ""),
f"scraped_{h}": s_entry.get(h, ""),
"truth_cores": list(t_entry.get("_cores", [])),
}
def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
+39 -2
View File
@@ -177,10 +177,13 @@ class EveryRevisionACoreAcceptsStays(unittest.TestCase):
def test_a_revision_known_by_one_hash_is_no_mismatch(self):
"""1.04 carries md5 and sha1, 1.07 only its sha1: a platform entry
giving 1.07's sha1 and md5 is that revision, not a mismatch."""
giving 1.07's sha1 and md5 is that revision, not a mismatch. dosbox
checks the sha1 (validation), so its md5 describes one revision and
contradicts nothing, the rule the export applies too."""
from truth import _hash_mismatch
truth = {"name": "MT32_CONTROL.ROM", "md5": "c" * 32, "sha1": ["a" * 40, "b" * 40]}
truth = {"name": "MT32_CONTROL.ROM", "md5": "c" * 32, "sha1": ["a" * 40, "b" * 40],
"validation": ["sha1"]}
scraped = {"name": "MT32_CONTROL.ROM", "md5": "d" * 32, "sha1": "b" * 40}
self.assertIsNone(_hash_mismatch(truth, scraped))
self.assertEqual(
@@ -189,6 +192,40 @@ class EveryRevisionACoreAcceptsStays(unittest.TestCase):
class DiffAndExportShareOneMismatchRule(unittest.TestCase):
"""diff_truth let one shared value settle every field and read no
sha256, while the export corrected each field on its own: the diff
stayed silent on hashes the export rewrote."""
CASES = (
({"sha1": "a" * 40, "md5": "1" * 32}, {"sha1": "a" * 40, "md5": "2" * 32}),
({"sha256": "a" * 64}, {"sha256": "b" * 64}),
({"sha1": "a" * 40, "md5": "1" * 32}, {"sha1": "a" * 40, "md5": "1" * 32}),
)
def test_the_diff_names_what_the_export_corrects(self):
from exporter.baseline import build_native_model # noqa: PLC0415
from truth import diff_platform_truth # noqa: PLC0415
for t_hashes, p_hashes in self.CASES:
with self.subTest(truth=t_hashes, platform=p_hashes):
truth = {"systems": {"s": {"files": [
{"name": "f.bin", "required": True, "_cores": ["c"], **t_hashes}]}}}
scraped = {"systems": {"s": {"files": [
{"name": "f.bin", "destination": "f.bin", "required": True, **p_hashes}]}}}
diff = diff_platform_truth(truth, scraped)
reported = {
m["hash_type"]
for system in diff.get("divergences", {}).values()
for m in system.get("hash_mismatch", [])
}
_, report = build_native_model(truth, scraped)
corrected = {item.split()[-1] for item in report.hashes_corrected}
# One record per entry, naming the first field the export corrects.
self.assertEqual(bool(reported), bool(corrected))
self.assertLessEqual(reported, corrected)
def _filed(truth: dict, name: str) -> list[str]:
"""The systems an entry of that name was filed under."""
return sorted(