From 9a77ce4a348c40bd6da7c000988d8346688227be Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:53:24 +0200 Subject: [PATCH] fix: split multi-hash strings in the truth diff --- scripts/truth.py | 54 ++++++++++++++++++++++------------------ tests/test_truth_diff.py | 22 ++++++++++++++++ 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/scripts/truth.py b/scripts/truth.py index d92cafd7..197b6c5a 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -9,6 +9,7 @@ from __future__ import annotations import sys from common import _norm_system_id, resolve_platform_cores, runs_standalone +from hashing import parse_md5_list from validation import filter_files_by_mode, read_from_system_dir @@ -535,6 +536,17 @@ def generate_platform_truth( # Platform truth diffing +def _hash_values(entry: dict, field: str) -> set[str]: + """The values a diffed entry declares for a hash field. + + A profile writes a list, a platform YAML one comma-separated string + (Recalbox's multi-hash): both are read the way the exporter reads them. + A number is a hash YAML coerced and no longer spells: it is ignored. + """ + raw = entry.get(field) + return set(parse_md5_list(raw)) if isinstance(raw, (str, list)) else set() + + def _match_renames( unmatched_truth: list[dict], unmatched_scraped: dict ) -> tuple[set[int], set]: @@ -560,31 +572,29 @@ def _match_renames( truth_hash_index: dict[str, int] = {} for position, fe in enumerate(unmatched_truth): for h in ("sha1", "md5", "crc32"): - val = fe.get(h) - if val and isinstance(val, str): - truth_hash_index[val.lower()] = position + for val in _hash_values(fe, h): + truth_hash_index[val] = position for s_key, s_entry in unmatched_scraped.items(): - for h in ("sha1", "md5", "crc32"): - s_val = s_entry.get(h) - if not s_val or not isinstance(s_val, str): - continue - position = truth_hash_index.get(s_val.lower()) - if position is not None: - # Rename detected. Count as matched - rename_matched_truth.add(position) - rename_matched_scraped.add(s_key) - break + position = next( + ( + truth_hash_index[val] + for h in ("sha1", "md5", "crc32") + for val in sorted(_hash_values(s_entry, h)) + if val in truth_hash_index + ), + None, + ) + if position is not None: + # Rename detected. Count as matched + rename_matched_truth.add(position) + rename_matched_scraped.add(s_key) return rename_matched_truth, rename_matched_scraped def _hash_set(entry: dict) -> set[str]: - values: set[str] = set() - for h in ("sha1", "md5", "crc32"): - value = entry.get(h) or [] - values.update(str(v).lower() for v in (value if isinstance(value, list) else [value])) - return values + return {value for h in ("sha1", "md5", "crc32") for value in _hash_values(entry, h)} def _path_tail(value: object) -> str: @@ -612,12 +622,8 @@ def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None: for h in ("sha1", "md5", "crc32"): t_hash = t_entry.get(h, "") s_hash = s_entry.get(h, "") - if not t_hash or not s_hash: - continue - # Normalize to list for multi-hash support - t_list = t_hash if isinstance(t_hash, list) else [t_hash] - s_list = s_hash if isinstance(s_hash, list) else [s_hash] - if not {v.lower() for v in t_list} & {v.lower() for v in s_list}: + t_values, s_values = _hash_values(t_entry, h), _hash_values(s_entry, h) + if t_values and s_values and not t_values & s_values: return { "name": s_entry["name"], "hash_type": h, diff --git a/tests/test_truth_diff.py b/tests/test_truth_diff.py index de462aa4..28cd2011 100644 --- a/tests/test_truth_diff.py +++ b/tests/test_truth_diff.py @@ -203,6 +203,28 @@ class RenameMatching(unittest.TestCase): +class PlatformMultiHash(unittest.TestCase): + """Recalbox and RetroDECK write accepted md5s as one comma string. + + Read as one value, scph1001.bin was a mismatch the export (which splits + them) never corrected: diff_truth reported 15 for Recalbox, 9 for + RetroDECK, against the 4 and 3 the content supports. + """ + + TRUTH = "924e392ed05558ffdb115408c263dccf" + SCRAPED = "dc2b9bf8da62ec93e868cfd29f0d067d,924e392ed05558ffdb115408c263dccf" + + def test_a_shared_value_is_no_mismatch(self): + truth = {"files": [{"name": "scph1001.bin", "md5": self.TRUTH}]} + scraped = {"files": [{"name": "scph1001.bin", "md5": self.SCRAPED}]} + self.assertEqual(_diff_system(truth, scraped), {}) + + def test_a_rename_is_found_through_the_list(self): + truth = [_entry("scph1001.bin", md5=self.TRUTH)] + scraped = {"psx.bin": _entry("psx.bin", md5=self.SCRAPED)} + self.assertEqual(_match_renames(truth, scraped), ({0}, {"psx.bin"})) + + class OnlySystemDirectoryEntries(unittest.TestCase): """The truth kept files read from the save, content or working directory: diff_truth listed Cemu's mlc01 certificates, panda3ds's seeddb.bin and