From d7d6f3e4126958233f35c4cea1c10015b7a02f82 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:50:09 +0100 Subject: [PATCH] fix: one hash mismatch rule for diff and export diff_truth read sha1, md5 and crc32 and let one shared value settle the entry, while export_native corrected each of the four fields on its own. contradicted_hashes is now the single decision both read. --- scripts/exporter/baseline.py | 29 ++++++++++++++++--- scripts/truth.py | 55 +++++++++++------------------------- tests/test_truth_layout.py | 41 +++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 45 deletions(-) diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index c5a2c1221..5420c2efe 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -84,6 +84,26 @@ def comparable(entry: dict | None) -> dict: return {k: v for k, v in entry.items() if k != "md5"} +def contradicted_hashes(truth_entry: dict, platform_entry: dict | None) -> list[str]: + """The hash fields a platform entry declares against the truth. + + A field contradicts when both sides declare it without a value in + common, among the hashes the code checks and the platform values a + profile can be compared with. export_native corrects exactly these and + diff_truth reports them: the diff read three fields and let one shared + value settle all of them, so it stayed silent on an md5 or a sha256 + the export rewrote. + """ + checked = checked_hashes(truth_entry) or {} + theirs = comparable(platform_entry) + return [ + field_name for field_name in HASH_FIELDS + if (ours := set(_hash_values(checked, field_name))) + and (held := set(_hash_values(theirs, field_name))) + and not ours & held + ] + + def is_label(entry: dict) -> bool: """A name only content-scanning cores give: no core opens that file. @@ -595,17 +615,18 @@ def build_native_model( if matched is not None: matched.truth = truth_entry checked = checked_hashes(truth_entry) or {} + contradicted = contradicted_hashes(truth_entry, matched.platform) for field_name in HASH_FIELDS: - ours = set(_hash_values(checked, field_name)) if field_name == "md5" and matched.composite_md5: continue - theirs = set(_hash_values(comparable(matched.platform), field_name)) - if ours and theirs and not ours & theirs: + if field_name in contradicted: matched.corrections.append(field_name) report.hashes_corrected.append( f"{matched.native_system}/{matched.name} {field_name}" ) - elif ours and not theirs: + elif _hash_values(checked, field_name) and not _hash_values( + comparable(matched.platform), field_name + ): matched.filled.append(field_name) t_req = truth_entry.get("required") p_req = (matched.platform or {}).get("required") diff --git a/scripts/truth.py b/scripts/truth.py index bc1e3c98c..d463368ac 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -18,7 +18,7 @@ from common import ( resolve_platform_cores, standalone_set as platform_standalone_set, ) -from hashing import compute_hashes, parse_md5_list +from hashing import compute_hashes from validation import ( filter_files_by_mode, installed_with_emulator, @@ -878,53 +878,30 @@ def generate_platform_truth( # Platform truth diffing -def _hash_values(entry: dict, field: str) -> set[str]: - """The values a diffed entry declares for a hash field. - - A profile writes a list, a platform YAML one comma-separated string - (Recalbox's multi-hash): both are read the way the exporter reads them. - A number is a hash YAML coerced and no longer spells: it is ignored. - """ - raw = entry.get(field) - return set(parse_md5_list(raw)) if isinstance(raw, (str, list)) else set() - - -def _hash_set(entry: dict) -> set[str]: - return {value for h in ("sha1", "md5", "crc32") for value in _hash_values(entry, h)} - - def _path_tail(value: object) -> str: return str(value or "").replace("\\", "/").casefold() def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None: - """The first hash both sides declare without a value in common. + """The first hash field the export would correct, by the export's rule. - A value shared on any field settles it: the truth holds every revision a - core accepts, and a profile may give one revision's sha1 without its md5. - A hash the code does not check contradicts nothing (checked_hashes), and - a platform md5 that is a composite of a zip's members is no container - md5 (comparable), the rule pair_entries applies. + contradicted_hashes is the one decision: a hash the code does not check + contradicts nothing (checked_hashes), and a platform md5 that is a + composite of a zip's members is no container md5 (comparable). """ - from exporter.baseline import checked_hashes, comparable + from exporter.baseline import contradicted_hashes - s_entry = comparable(s_entry) - if _hash_set(t_entry) & _hash_set(s_entry): + fields = contradicted_hashes(t_entry, s_entry) + if not fields: return None - t_entry = checked_hashes(t_entry) or {} - for h in ("sha1", "md5", "crc32"): - t_hash = t_entry.get(h, "") - s_hash = s_entry.get(h, "") - t_values, s_values = _hash_values(t_entry, h), _hash_values(s_entry, h) - if t_values and s_values and not t_values & s_values: - return { - "name": s_entry["name"], - "hash_type": h, - f"truth_{h}": t_hash, - f"scraped_{h}": s_hash, - "truth_cores": list(t_entry.get("_cores", [])), - } - return None + h = fields[0] + return { + "name": s_entry["name"], + "hash_type": h, + f"truth_{h}": t_entry.get(h, ""), + f"scraped_{h}": s_entry.get(h, ""), + "truth_cores": list(t_entry.get("_cores", [])), + } def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict: diff --git a/tests/test_truth_layout.py b/tests/test_truth_layout.py index 4c187b261..50a23053b 100644 --- a/tests/test_truth_layout.py +++ b/tests/test_truth_layout.py @@ -177,10 +177,13 @@ class EveryRevisionACoreAcceptsStays(unittest.TestCase): def test_a_revision_known_by_one_hash_is_no_mismatch(self): """1.04 carries md5 and sha1, 1.07 only its sha1: a platform entry - giving 1.07's sha1 and md5 is that revision, not a mismatch.""" + giving 1.07's sha1 and md5 is that revision, not a mismatch. dosbox + checks the sha1 (validation), so its md5 describes one revision and + contradicts nothing, the rule the export applies too.""" from truth import _hash_mismatch - truth = {"name": "MT32_CONTROL.ROM", "md5": "c" * 32, "sha1": ["a" * 40, "b" * 40]} + truth = {"name": "MT32_CONTROL.ROM", "md5": "c" * 32, "sha1": ["a" * 40, "b" * 40], + "validation": ["sha1"]} scraped = {"name": "MT32_CONTROL.ROM", "md5": "d" * 32, "sha1": "b" * 40} self.assertIsNone(_hash_mismatch(truth, scraped)) self.assertEqual( @@ -189,6 +192,40 @@ class EveryRevisionACoreAcceptsStays(unittest.TestCase): +class DiffAndExportShareOneMismatchRule(unittest.TestCase): + """diff_truth let one shared value settle every field and read no + sha256, while the export corrected each field on its own: the diff + stayed silent on hashes the export rewrote.""" + + CASES = ( + ({"sha1": "a" * 40, "md5": "1" * 32}, {"sha1": "a" * 40, "md5": "2" * 32}), + ({"sha256": "a" * 64}, {"sha256": "b" * 64}), + ({"sha1": "a" * 40, "md5": "1" * 32}, {"sha1": "a" * 40, "md5": "1" * 32}), + ) + + def test_the_diff_names_what_the_export_corrects(self): + from exporter.baseline import build_native_model # noqa: PLC0415 + from truth import diff_platform_truth # noqa: PLC0415 + + for t_hashes, p_hashes in self.CASES: + with self.subTest(truth=t_hashes, platform=p_hashes): + truth = {"systems": {"s": {"files": [ + {"name": "f.bin", "required": True, "_cores": ["c"], **t_hashes}]}}} + scraped = {"systems": {"s": {"files": [ + {"name": "f.bin", "destination": "f.bin", "required": True, **p_hashes}]}}} + diff = diff_platform_truth(truth, scraped) + reported = { + m["hash_type"] + for system in diff.get("divergences", {}).values() + for m in system.get("hash_mismatch", []) + } + _, report = build_native_model(truth, scraped) + corrected = {item.split()[-1] for item in report.hashes_corrected} + # One record per entry, naming the first field the export corrects. + self.assertEqual(bool(reported), bool(corrected)) + self.assertLessEqual(reported, corrected) + + def _filed(truth: dict, name: str) -> list[str]: """The systems an entry of that name was filed under.""" return sorted(