diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index e3e3df6f..ad366382 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -35,6 +35,32 @@ def _hash_values(entry: dict, field_name: str) -> list[str]: return [v.strip().lower() for v in str(raw).split(",") if v.strip()] +def checked_hashes(entry: dict | None) -> dict | None: + """The truth entry with only the hashes its code checks. + + `validation:` names what the code verifies. Where a checked hash names + one value, the content is pinned and every hash describes it. Where + the code checks no hash, or accepts several values, a hash it does not + name describes one revision among those it accepts: dosbox checks + MT32_CONTROL.ROM against nine sha1, and the one md5 beside them + replaced System.dat's, another accepted revision. Such a value still + pairs a renamed file, it never corrects or fills one. + """ + if not entry or entry.get("validation") is None: + return entry + raw = entry["validation"] + if isinstance(raw, dict): + kinds = {str(k) for values in raw.values() for k in values or ()} + elif isinstance(raw, list): + kinds = {str(k) for k in raw} + else: + kinds = {str(raw)} + checked = [f for f in HASH_FIELDS if f in kinds and _hash_values(entry, f)] + if checked and all(len(set(_hash_values(entry, f))) == 1 for f in checked): + return entry + return {k: v for k, v in entry.items() if k not in HASH_FIELDS or k in kinds} + + def _is_placeholder(name: str) -> bool: return "<" in name or ">" in name or "*" in name @@ -76,6 +102,14 @@ class NativeFile: return "both" return "platform" if self.platform is not None else "truth" + @property + def _checked_truth(self) -> dict: + """The truth's hashes that can overrule the platform's. An addition + has no platform value to overrule and keeps them all.""" + if self.platform is None: + return self.truth or {} + return checked_hashes(self.truth) or {} + def _identity_source(self) -> str: """Which side's content fields describe the file: one side, or both. @@ -86,7 +120,7 @@ class NativeFile: truth's crc32 beside Batocera's md5 for bios7.bin is one dump, and reading it as a contradiction emptied the md5 the frontend checks. """ - truth, plat = _field_sets(self.truth), _field_sets(self.platform) + truth, plat = _field_sets(self._checked_truth), _field_sets(self.platform) if not any(truth.values()): return "platform" if not any(plat.values()): @@ -109,7 +143,7 @@ class NativeFile: the divergence is recorded, never silently merged: an emulator that rejects a file will reject it whatever the platform declares. """ - truth_values = _hash_values(self.truth or {}, field_name) + truth_values = _hash_values(self._checked_truth, field_name) platform_values = _hash_values(self.platform or {}, field_name) source = self._identity_source() if source == "platform": @@ -139,7 +173,7 @@ class NativeFile: is written when it names one value, which is then that dump's. """ source = self._identity_source() - truth_values = _hash_values(self.truth or {}, field_name) + truth_values = _hash_values(self._checked_truth, field_name) platform_values = _hash_values(self.platform or {}, field_name) if source == "truth": return truth_values[0] if truth_values else "" @@ -491,8 +525,9 @@ def build_native_model( if matched is not None: matched.truth = truth_entry + checked = checked_hashes(truth_entry) or {} for field_name in HASH_FIELDS: - ours = set(_hash_values(truth_entry, field_name)) + ours = set(_hash_values(checked, field_name)) theirs = set(_hash_values(matched.platform or {}, field_name)) if ours and theirs and not ours & theirs: matched.corrections.append(field_name) diff --git a/scripts/truth.py b/scripts/truth.py index 9217a618..d02f3022 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -602,9 +602,13 @@ def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None: A value shared on any field settles it: the truth holds every revision a core accepts, and a profile may give one revision's sha1 without its md5. + A hash the code does not check contradicts nothing (checked_hashes). """ + from exporter.baseline import checked_hashes + if _hash_set(t_entry) & _hash_set(s_entry): return None + t_entry = checked_hashes(t_entry) or {} for h in ("sha1", "md5", "crc32"): t_hash = t_entry.get(h, "") s_hash = s_entry.get(h, "") diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 19502e5a..c616ad62 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -258,6 +258,75 @@ class Reconciliation(unittest.TestCase): self.assertEqual(added.destination, "fuse/128-1.rom") self.assertEqual(report.files_added, 1) + def test_a_hash_the_code_does_not_check_corrects_nothing(self): + """dosbox checks MT32_CONTROL.ROM by sha1 and size. The truth accepts + System.dat's sha1 but carries the md5 and crc of another revision, + and the export wrote those beside System.dat's sha1: a record no + file satisfies, counted as a correction.""" + truth = {"systems": {"dos": {"files": [ + {"name": "MT32_CONTROL.ROM", "sha1": ["1" * 40, "2" * 40], + "md5": "a" * 32, "crc32": "aaaaaaaa", "size": 65536, + "validation": ["sha1", "size"]}, + ]}}} + scraped = {"systems": {"dos": {"native_id": "dos", "files": [ + {"name": "MT32_CONTROL.ROM", "destination": "MT32_CONTROL.ROM", + "native_system": "dos", "sha1": "2" * 40, "md5": "b" * 32, + "crc32": "bbbbbbbb", "size": 65536}, + ]}}} + systems, report = build_native_model(truth, scraped) + entry = systems["dos"].files[0] + self.assertEqual( + [entry.record_hash(f) for f in ("crc32", "md5", "sha1")], + ["bbbbbbbb", "b" * 32, "2" * 40], + ) + self.assertEqual(entry.hashes("md5"), ["b" * 32]) + self.assertEqual(report.hashes_corrected, []) + + def test_a_size_checked_file_keeps_the_dump_the_platform_names(self): + """pcsx_rearmed and DuckStation check scph1001.bin by size. The + truth's md5 is v4.1's, Batocera names v2.0: the export replaced it + and the frontend then refused the v2.0 the cores load. Nor is a + blank md5 filled with one revision the code does not require.""" + truth = {"systems": {"sony-playstation": {"files": [ + {"name": "scph1001.bin", "md5": "a" * 32, "size": 524288, + "validation": ["size"]}, + {"name": "scph5500.bin", "md5": "c" * 32, "size": 524288, + "validation": ["size"]}, + ]}}} + scraped = {"systems": {"sony-playstation": {"native_id": "psx", "files": [ + {"name": "scph1001.bin", "destination": "scph1001.bin", + "native_system": "psx", "md5": "b" * 32}, + {"name": "scph5500.bin", "destination": "scph5500.bin", + "native_system": "psx"}, + ]}}} + systems, report = build_native_model(truth, scraped) + v20, blank = systems["psx"].files + self.assertEqual(v20.hash("md5"), "b" * 32) + self.assertEqual(blank.hash("md5"), "") + self.assertEqual((report.hashes_corrected, blank.filled), ([], [])) + + def test_a_hash_beside_one_checked_value_still_corrects(self): + """A core that checks one crc32 pins one dump, and the md5 of that + dump is no less true: a platform md5 against it is corrected.""" + truth = {"systems": {"coco": {"files": [ + {"name": "bas12.rom", "crc32": "aaaaaaaa", "md5": "a" * 32, + "validation": ["crc32"]}, + ]}}} + scraped = {"systems": {"coco": {"native_id": "coco", "files": [ + {"name": "bas12.rom", "destination": "bas12.rom", + "native_system": "coco", "md5": "b" * 32}, + ]}}} + systems, report = build_native_model(truth, scraped) + self.assertEqual(systems["coco"].files[0].hash("md5"), "a" * 32) + self.assertEqual(report.hashes_corrected, ["coco/bas12.rom md5"]) + + def test_the_diff_names_no_mismatch_the_code_does_not_check(self): + from truth import _diff_system + truth_sys = {"files": [{"name": "scph1001.bin", "md5": "a" * 32, + "validation": ["size"]}]} + scraped_sys = {"files": [{"name": "scph1001.bin", "md5": "b" * 32}]} + self.assertNotIn("hash_mismatch", _diff_system(truth_sys, scraped_sys)) + def test_the_diff_pairs_by_the_same_rule(self): """diff_truth reads the same pairing, or it names a mismatch the export does not correct."""