diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index c5a2c1221..5420c2efe 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -84,6 +84,26 @@ def comparable(entry: dict | None) -> dict: return {k: v for k, v in entry.items() if k != "md5"} +def contradicted_hashes(truth_entry: dict, platform_entry: dict | None) -> list[str]: + """The hash fields a platform entry declares against the truth. + + A field contradicts when both sides declare it without a value in + common, among the hashes the code checks and the platform values a + profile can be compared with. export_native corrects exactly these and + diff_truth reports them: the diff read three fields and let one shared + value settle all of them, so it stayed silent on an md5 or a sha256 + the export rewrote. + """ + checked = checked_hashes(truth_entry) or {} + theirs = comparable(platform_entry) + return [ + field_name for field_name in HASH_FIELDS + if (ours := set(_hash_values(checked, field_name))) + and (held := set(_hash_values(theirs, field_name))) + and not ours & held + ] + + def is_label(entry: dict) -> bool: """A name only content-scanning cores give: no core opens that file. @@ -595,17 +615,18 @@ def build_native_model( if matched is not None: matched.truth = truth_entry checked = checked_hashes(truth_entry) or {} + contradicted = contradicted_hashes(truth_entry, matched.platform) for field_name in HASH_FIELDS: - ours = set(_hash_values(checked, field_name)) if field_name == "md5" and matched.composite_md5: continue - theirs = set(_hash_values(comparable(matched.platform), field_name)) - if ours and theirs and not ours & theirs: + if field_name in contradicted: matched.corrections.append(field_name) report.hashes_corrected.append( f"{matched.native_system}/{matched.name} {field_name}" ) - elif ours and not theirs: + elif _hash_values(checked, field_name) and not _hash_values( + comparable(matched.platform), field_name + ): matched.filled.append(field_name) t_req = truth_entry.get("required") p_req = (matched.platform or {}).get("required") diff --git a/scripts/exporter/misterfpga_exporter.py b/scripts/exporter/misterfpga_exporter.py index 1567df50e..c84fc7ad4 100644 --- a/scripts/exporter/misterfpga_exporter.py +++ b/scripts/exporter/misterfpga_exporter.py @@ -86,8 +86,14 @@ class Exporter(BaseExporter): return super().states(fe, field_name) def written(self, fe: NativeFile, field_name: str) -> str: - """An entry is checked against one md5, the first accepted.""" - return fe.hash(field_name) + """An entry is one record: its md5, size and url describe one dump. + + The first value the truth accepts replaced MiSTer's own accepted + md5 while the url still served MiSTer's dump, which the Downloader + then refused (fetch_file_worker.py:127), and the report said + nothing. record_hash keeps the platform's value when it is accepted. + """ + return fe.record_hash(field_name) def _by_path(self, systems: dict[str, NativeSystem]) -> dict[str, object]: indexed: dict[str, object] = {} @@ -119,7 +125,7 @@ class Exporter(BaseExporter): fe = indexed.get(path) if fe is None: continue - md5 = fe.hash("md5") + md5 = self.written(fe, "md5") if not md5: # The record is checked by its md5: where the truth takes the # entry without one, a new size beside the old hash describes @@ -185,7 +191,7 @@ class Exporter(BaseExporter): indexed = self._by_path(systems) for path, fe in indexed.items(): declared = files.get(path) - md5 = fe.hash("md5") + md5 = self.written(fe, "md5") if ( declared is not None and md5 and declared.get("hash") != md5 and path not in getattr(self, "_unsized", []) diff --git a/scripts/exporter/retropie_exporter.py b/scripts/exporter/retropie_exporter.py index 0847f7a7a..7e31af28a 100644 --- a/scripts/exporter/retropie_exporter.py +++ b/scripts/exporter/retropie_exporter.py @@ -156,6 +156,30 @@ class Exporter(BaseExporter): @classmethod def _enumeration(cls, help_text: str) -> tuple[int, int, list[str]] | None: + """The first BIOS clause's list of names to extend, or None.""" + clauses = cls._clauses(help_text) + return clauses[0][2] if clauses else None + + @classmethod + def _clauses( + cls, help_text: str + ) -> list[tuple[int, list[str], tuple[int, int, list[str]] | None]]: + """Each BIOS clause that names files: its start, its names, and the + list to extend or None when it cannot be extended. + + flycast's help has a Dreamcast clause and a Naomi/Atomiswave one. + Only the first was read, and naomi2.zip and segasp.zip were added to + the Dreamcast list. + """ + found = [] + for clause in _BIOS_CLAUSE.finditer(help_text): + names = [m.group(0) for m in cls._names_in(clause.group(0))] + if names: + found.append((clause.start(), names, cls._extendable(clause))) + return found + + @classmethod + def _extendable(cls, clause: re.Match[str]) -> tuple[int, int, list[str]] | None: """The list of names to extend: its span and its names, or None. A list of alternatives ("a or b", "a/b") is not an enumeration: an @@ -164,9 +188,6 @@ class Exporter(BaseExporter): are a list of examples ("eg. a, b") and one with words between its names, which a rewrite would lose. """ - clause = _BIOS_CLAUSE.search(help_text) - if clause is None: - return None text = clause.group(0) names = cls._names_in(text) if not names: @@ -308,13 +329,12 @@ class Exporter(BaseExporter): by_core = self._files_by_core(systems) # Aliases count as names we know: RetroPie writes dc_flash.bin where # flycast's profile files it as an alias of another primary. - known: set[str] = set() + systems_of: dict[str, set[str]] = {} for system in systems.values(): for fe in system.files: - known.add(fe.name.lower()) - known.update( - str(a).lower() for a in (fe.native("aliases", []) or []) - ) + for name in [fe.name, *(fe.native("aliases", []) or [])]: + systems_of.setdefault(str(name).lower(), set()).add(fe.native_system) + known = set(systems_of) self._skipped: dict[str, list[str]] = {} self._named: set[int] = set() produced: dict[str, str] = {} @@ -362,30 +382,52 @@ class Exporter(BaseExporter): and (slot := self._slot_of(fe)) is not None } proposed = self._one_per_slot(candidates, filled) - missing = self._in_search_order(proposed) - if not missing: + if not proposed: continue - enumeration = self._enumeration(help_text) - if enumeration is None: - # No enumeration to extend, and a sentence we would have to - # write ourselves is a documentation change, not a correction. - skip("whose help text lists no files to add to", module_id) + # A file joins the list of its own system: the clause whose + # names the platform files under that system, and only one. + clauses = self._clauses(help_text) + homes: dict[int, list[NativeFile]] = {} + homeless = False + for fe in proposed: + owners = [ + i for i, (_, names, _) in enumerate(clauses) + if fe.native_system in set().union( + *(systems_of.get(n.lower(), set()) for n in names)) + ] + if len(owners) == 1 and clauses[owners[0]][2] is not None: + homes.setdefault(owners[0], []).append(fe) + else: + homeless = True + if homeless: + # No enumeration of that system to extend, and a sentence we + # would have to write ourselves is a documentation change, + # not a correction. + skip("whose help text lists no files of that system to add to", module_id) + if not homes: continue - if len(listed) + len(missing) > self._longest: + added = sum(len(self._in_search_order(fes)) for fes in homes.values()) + if len(listed) + added > self._longest: skip("whose help text would grow longer than any it has today", module_id) continue - new_help = self._extended(help_text, enumeration, missing) - if len(listed) + len(missing) > 1: - new_help = _BIOS_NOUN.sub(r"\1 files", new_help, count=1) + new_help = help_text + for i in sorted(homes, reverse=True): + start, _, enumeration = clauses[i] + missing = self._in_search_order(homes[i]) + new_help = self._extended(new_help, enumeration, missing) + if len(enumeration[2]) + len(missing) > 1: + new_help = new_help[:start] + _BIOS_NOUN.sub( + r"\1 files", new_help[start:enumeration[0]], count=1 + ) + new_help[enumeration[0]:] + self._named.update(id(fe) for fe in homes[i]) produced[relative] = originals[relative].replace( f'rp_module_help="{help_text}"', f'rp_module_help="{new_help}"', 1, ) - self._named.update(id(fe) for fe in proposed) return produced diff --git a/scripts/truth.py b/scripts/truth.py index 3488136d9..d463368ac 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -18,7 +18,7 @@ from common import ( resolve_platform_cores, standalone_set as platform_standalone_set, ) -from hashing import compute_hashes, parse_md5_list +from hashing import compute_hashes from validation import ( filter_files_by_mode, installed_with_emulator, @@ -113,7 +113,6 @@ _FILLED_FIELDS = ( "note", "aliases", "contents", - "region", ) _COPIED_FIELDS = ( "sha1", @@ -290,6 +289,27 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None: if file_entry.get(field) is not None and existing.get(field) is None: existing[field] = file_entry[field] _merge_priority(existing, file_entry.get("priority")) + _merge_region(existing, file_entry.get("region")) + + +def _merge_region(existing: dict, theirs: str | list | None) -> None: + """The regions every reader selects the file for, the pack's own rule. + + region.build_region_index takes the union of the tags and keeps a file + untagged as soon as one reader declares it without a region. The first + tag used to win: psxonpsp660.bin came out japan from pcsx_rearmed while + SwanStation tags it world and beetle_psx not at all. + """ + if existing.get("_region_void"): + return + if not theirs: + existing.pop("region", None) + existing["_region_void"] = True + return + ours = existing.get("region") or [] + ours = ours if isinstance(ours, list) else [ours] + added = [r for r in (theirs if isinstance(theirs, list) else [theirs]) if r not in ours] + existing["region"] = [*ours, *added] def _merge_hashes( @@ -373,6 +393,8 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict: crc = entry.get("crc32") if isinstance(crc, str): entry["crc32"] = crc.removeprefix("0x").lower() + if not file_entry.get("region"): + entry["_region_void"] = True entry["_cores"] = {emu_name} entry["_required_by"] = {emu_name} if file_entry.get("required") else set() entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set() @@ -486,14 +508,19 @@ def _sizes_clash(a: dict, b: dict) -> bool: return False -def _identity_values(entry: dict, db: dict | None) -> set[str]: - """Every hash an entry declares, with the siblings the database knows.""" +def _identity(entry: dict, db: dict | None) -> dict: + """The entry with the hashes and size the database knows for its hash.""" probe = {k: entry[k] for k in ("sha1", "md5", "crc32", "size") if entry.get(k)} if db and probe and not any(isinstance(v, list) for v in probe.values()): _enrich_hashes(probe, db) + return {**entry, **{k: v for k, v in probe.items() if not entry.get(k)}} + + +def _identity_values(entry: dict) -> set[str]: + """Every hash an identity declares.""" values: set[str] = set() for field in ("sha1", "md5", "sha256", "crc32"): - values |= _declared(entry, field) | _declared(probe, field) + values |= _declared(entry, field) return values @@ -503,10 +530,14 @@ def _declaration_verdict(entry: dict, declaration: dict, db: dict | None) -> str A shared hash proves it. Sizes no value of which agrees refute it, as name_match_size_ok refutes a candidate. Two hashes at one size are two revisions of one slot, as slots.py reads them, so they refute nothing. + The size the database holds for a declared hash counts as declared: + Batocera's BK BASIC10.ROM names only an md5, of a 24576-byte file, and + clk's 16 KB Oric basic10.rom was filed under the BK beside it. """ - if _identity_values(entry, db) & _identity_values(declaration, db): + ours, theirs = _identity(entry, db), _identity(declaration, db) + if _identity_values(ours) & _identity_values(theirs): return "same" - if _sizes_clash(entry, declaration): + if _sizes_clash(ours, theirs): return "other" return "undecided" @@ -814,6 +845,7 @@ def generate_platform_truth( for sys_id, sys_data in systems.items(): for fe in sys_data.get("files", []): fe.pop("_hash_by", None) + fe.pop("_region_void", None) fe["_cores"] = sorted(fe.get("_cores", set())) fe["_required_by"] = sorted(fe.get("_required_by", set())) fe["_named_by"] = sorted(fe.get("_named_by", set())) @@ -846,53 +878,30 @@ def generate_platform_truth( # Platform truth diffing -def _hash_values(entry: dict, field: str) -> set[str]: - """The values a diffed entry declares for a hash field. - - A profile writes a list, a platform YAML one comma-separated string - (Recalbox's multi-hash): both are read the way the exporter reads them. - A number is a hash YAML coerced and no longer spells: it is ignored. - """ - raw = entry.get(field) - return set(parse_md5_list(raw)) if isinstance(raw, (str, list)) else set() - - -def _hash_set(entry: dict) -> set[str]: - return {value for h in ("sha1", "md5", "crc32") for value in _hash_values(entry, h)} - - def _path_tail(value: object) -> str: return str(value or "").replace("\\", "/").casefold() def _hash_mismatch(t_entry: dict, s_entry: dict) -> dict | None: - """The first hash both sides declare without a value in common. + """The first hash field the export would correct, by the export's rule. - A value shared on any field settles it: the truth holds every revision a - core accepts, and a profile may give one revision's sha1 without its md5. - A hash the code does not check contradicts nothing (checked_hashes), and - a platform md5 that is a composite of a zip's members is no container - md5 (comparable), the rule pair_entries applies. + contradicted_hashes is the one decision: a hash the code does not check + contradicts nothing (checked_hashes), and a platform md5 that is a + composite of a zip's members is no container md5 (comparable). """ - from exporter.baseline import checked_hashes, comparable + from exporter.baseline import contradicted_hashes - s_entry = comparable(s_entry) - if _hash_set(t_entry) & _hash_set(s_entry): + fields = contradicted_hashes(t_entry, s_entry) + if not fields: return None - t_entry = checked_hashes(t_entry) or {} - for h in ("sha1", "md5", "crc32"): - t_hash = t_entry.get(h, "") - s_hash = s_entry.get(h, "") - t_values, s_values = _hash_values(t_entry, h), _hash_values(s_entry, h) - if t_values and s_values and not t_values & s_values: - return { - "name": s_entry["name"], - "hash_type": h, - f"truth_{h}": t_hash, - f"scraped_{h}": s_hash, - "truth_cores": list(t_entry.get("_cores", [])), - } - return None + h = fields[0] + return { + "name": s_entry["name"], + "hash_type": h, + f"truth_{h}": t_entry.get(h, ""), + f"scraped_{h}": s_entry.get(h, ""), + "truth_cores": list(t_entry.get("_cores", [])), + } def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict: diff --git a/scripts/validate_pr.py b/scripts/validate_pr.py index 7932c84ee..f0a6f9d5e 100644 --- a/scripts/validate_pr.py +++ b/scripts/validate_pr.py @@ -34,6 +34,8 @@ from common import ( parse_md5_list, ) from hashing import parse_hash_list +from nativemode import digest_algorithm +from validation import frontend_digest_matches, inner_rom_check try: import yaml @@ -140,7 +142,8 @@ def load_platform_hashes(platforms_dir: str, db: dict | None = None) -> dict: bios/// folders the collection files its system under, read from the other files of that system it holds. """ - known = {"sha1": set(), "md5": set(), "names": set(), "folders": {}, "sizes": {}} + known = {"sha1": set(), "md5": set(), "sha256": set(), "crc32": set(), + "names": set(), "folders": {}, "sizes": {}, "entries": {}} systems_of: dict[str, set[str]] = {} folders_of: dict[str, dict[str, int]] = {} files = (db or {}).get("files", {}) @@ -160,15 +163,31 @@ def load_platform_hashes(platforms_dir: str, db: dict | None = None) -> dict: print(f"WARNING: {name}: {exc}", file=sys.stderr) continue + mode, rule = config.get("verification_mode"), config.get("hash_rule", "") for system_id, system in config.get("systems", {}).items(): for file_entry in system.get("files", []): sha1s = parse_hash_list(file_entry.get("sha1")) md5s = parse_md5_list(file_entry.get("md5")) + sha256s = parse_hash_list(file_entry.get("sha256")) known["sha1"].update(sha1s) known["md5"].update(md5s) + known["sha256"].update(sha256s) + if not (sha1s or md5s or sha256s) and isinstance(file_entry.get("size"), int): + # The resolver's reading: a crc32 counts where no stronger + # hash is declared, bounded by the declared size. + known["crc32"].update( + (crc.zfill(8), file_entry["size"]) + for crc in parse_hash_list(file_entry.get("crc32")) + ) if "name" in file_entry: known["names"].add(file_entry["name"]) _note_sizes(known["sizes"], [file_entry["name"]], file_entry) + # A frontend that hashes a ZIP's member or its members + # together, or that takes any hash at the declared size, + # answers by its own rule, read per entry. + known["entries"].setdefault(file_entry["name"].lower(), []).append( + (file_entry, mode, rule, name) + ) for value in (*sha1s, *md5s): systems_of.setdefault(value, set()).add(system_id) _note_folder(folders_of, system_id, files, by_md5, sha1s, md5s) @@ -354,6 +373,14 @@ def validate_file( sha1_known = result.sha1 in platform_hashes.get("sha1", set()) md5_known = result.md5 in platform_hashes.get("md5", set()) + other_known = ( + hashes.get("sha256", "") in platform_hashes.get("sha256", set()) + or (result.crc32, result.size) in platform_hashes.get("crc32", set()) + ) + accepted_by = ( + set() if sha1_known or md5_known or other_known + else _platforms_accepting(filepath, result.filename, platform_hashes) + ) name_known = result.filename in platform_hashes.get("names", set()) profile_hashes = profile_hashes or {} @@ -369,6 +396,12 @@ def validate_file( result.add_check(True, "SHA1 matches known platform requirement") elif md5_known: result.add_check(True, "MD5 matches known platform requirement") + elif other_known: + result.add_check(True, "Hash matches known platform requirement") + elif accepted_by: + result.add_check( + True, "Accepted by the platform's own check: " + ", ".join(sorted(accepted_by)) + ) elif by_profile: names = sorted(by_profile) shown = ", ".join(names[:5]) + (f" and {len(names) - 5} more" if len(names) > 5 else "") @@ -415,10 +448,10 @@ def validate_file( "File is not under bios/ yet. Its system's files are in " + " or ".join(f"`{f}`" for f in expected_folders) ) - elif _variant_home(db, result.filename): + elif _variant_home(db, result.filename, result.sha1): result.add_warning( "File is not under bios/ yet. The collection keeps this name in " - f"`{_variant_home(db, result.filename)}`" + f"`{_variant_home(db, result.filename, result.sha1)}`" ) else: result.add_warning( @@ -439,7 +472,8 @@ def validate_file( f"this file has {result.size:,}: truncated or another file" ) - if name_known and not sha1_known and not md5_known and not by_profile and size_fits: + if (name_known and not (sha1_known or md5_known or other_known or accepted_by) + and not by_profile and size_fits): home = _variant_home(db, result.filename) result.add_info( "This may be a valid variant. If accepted, it will be placed in " @@ -449,10 +483,43 @@ def validate_file( return result -def _variant_home(db: dict | None, name: str) -> str: - """The folder the collection keeps a name in, for its variants beside it.""" +def _platforms_accepting(filepath: str, filename: str, platform_hashes: dict) -> set[str]: + """Platforms whose declaration of this name accepts the file by their + own check, as verify and the pack builder read it. + + Batocera pins a ROM inside the ZIP (zipped_file), Recalbox hashes a + ZIP's members together, RomM takes any hash at the declared size: the + container's sha1 or md5 is in none of those lists, and adam_fdc.zip, + which Batocera accepts, was called an unknown variant. + """ + accepted: set[str] = set() + for entry, mode, rule, platform in platform_hashes.get("entries", {}).get( + filename.lower(), [] + ): + if platform in accepted: + continue + if entry.get("zipped_file"): + ok = inner_rom_check(entry, filepath) == "ok" + else: + algorithm = digest_algorithm(mode) + ok = bool(algorithm or rule) and frontend_digest_matches( + entry, filepath, algorithm or "", rule + ) + if ok: + accepted.add(platform) + return accepted + + +def _variant_home(db: dict | None, name: str, held: str = "") -> str: + """The folder the collection keeps a name in, for its variants beside it. + + The folder of an identical copy comes first: adam_fdc.zip, held in + bios/Coleco/Adam/, was sent to bios/Arcade/MAME/, the first folder + the name index gave. + """ files = (db or {}).get("files", {}) - for sha1 in (db or {}).get("indexes", {}).get("by_name", {}).get(name, []): + by_name = (db or {}).get("indexes", {}).get("by_name", {}).get(name, []) + for sha1 in ([held] if held in files else []) + list(by_name): path = str(files.get(sha1, {}).get("path", "")) folder = path.rsplit("/", 1)[0] + "/" if "/" in path else "" if folder and "/.variants" not in "/" + folder: diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 108da1187..f84ee542f 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -737,6 +737,40 @@ class SearchOrder(unittest.TestCase): self.assertNotIn("us_scd2_9306", produced) self.assertNotIn("jp_mcd1_9112", produced) + def test_a_file_joins_the_list_of_its_own_system(self): + """flycast's help names the Dreamcast BIOS, then the Naomi/Atomiswave + ones. naomi2.zip, an arcade archive, went into the Dreamcast list: + only the first BIOS clause was read.""" + from exporter.baseline import NativeSystem # noqa: PLC0415 + + def entry(name, system): + return NativeFile(name=name, destination=f"dc/{name}", native_system=system, + truth={"name": name, "required": True, "md5": "0" * 32, + "_cores": ["flycast"], "_required_by": ["flycast"]}) + + dc, arcade = "Sega - Dreamcast", "Sega - Dreamcast-based Arcade" + systems = { + dc: NativeSystem(dc, files=[entry("dc_boot.bin", dc)]), + arcade: NativeSystem(arcade, files=[entry("naomi.zip", arcade), + entry("awbios.zip", arcade), + entry("naomi2.zip", arcade)]), + "other": NativeSystem("other", files=[entry("other.zip", "other")]), + } + script = ('rp_module_id="lr-flycast"\nrp_module_help="Copy the required Dreamcast ' + 'BIOS file dc_boot.bin to $biosdir/dc\\n\\nCopy the required ' + 'Naomi/Atomiswave BIOS files naomi.zip and awbios.zip to $biosdir/dc"\n') + longest = ('rp_module_id="xm7"\nrp_module_help="Copy bios files ' + + ", ".join(f"n{i}.rom" for i in range(12)) + ' and n12.rom to $biosdir"\n') + exporter = discover_exporters()["retropie"]() + produced = exporter.render( + systems, None, + {"scriptmodules/libretrocores/lr-flycast.sh": script, + "scriptmodules/emulators/xm7.sh": longest}, + )["scriptmodules/libretrocores/lr-flycast.sh"] + self.assertIn("Dreamcast BIOS file dc_boot.bin to", produced) + self.assertIn("BIOS files naomi.zip, awbios.zip and naomi2.zip to", produced) + self.assertNotIn("other.zip", produced) + def test_cores_that_disagree_keep_the_best_rank(self): """Nothing is dropped on the rank here, so the best one wins. @@ -1849,6 +1883,27 @@ class MisterDatabaseComesBackAsWritten(unittest.TestCase): self.assertEqual(len(exporter.withheld()), 1) self.assertEqual(exporter.validate(systems, produced), []) + def test_an_accepted_hash_is_not_swapped_for_another(self): + """The url serves MiSTer's dump, and the truth accepts it among two. + The first accepted md5 went in its place with no change reported, + and the Downloader refused every install of the file.""" + database = {"db_id": "bios_db", "files": { + "games/X/boot.rom": {"hash": "b" * 32, "size": 10, "url": "u"}, + }} + original = json.dumps(database) + scraped = {"systems": {"x": {"files": [{ + "name": "boot.rom", "destination": "X/boot.rom", "md5": "b" * 32, "size": 10, + }]}}} + truth = {"systems": {"x": {"files": [{ + "name": "boot.rom", "path": "X/boot.rom", "md5": ["a" * 32, "b" * 32], + "size": 10, "_cores": ["x"], + }]}}} + exporter = discover_exporters()["misterfpga"]() + systems, report = build_native_model(truth, scraped) + produced = exporter.render(systems, report, {"bios_db.json": original}, scraped) + self.assertEqual(produced["bios_db.json"], original) + self.assertEqual(exporter.validate(systems, produced), []) + def test_an_empty_truth_hands_back_the_database(self): archive = UPSTREAM_CACHE / "misterfpga" / "bios_db.json.zip" if not archive.exists(): diff --git a/tests/test_truth_layout.py b/tests/test_truth_layout.py index 73f514d28..50a23053b 100644 --- a/tests/test_truth_layout.py +++ b/tests/test_truth_layout.py @@ -177,10 +177,13 @@ class EveryRevisionACoreAcceptsStays(unittest.TestCase): def test_a_revision_known_by_one_hash_is_no_mismatch(self): """1.04 carries md5 and sha1, 1.07 only its sha1: a platform entry - giving 1.07's sha1 and md5 is that revision, not a mismatch.""" + giving 1.07's sha1 and md5 is that revision, not a mismatch. dosbox + checks the sha1 (validation), so its md5 describes one revision and + contradicts nothing, the rule the export applies too.""" from truth import _hash_mismatch - truth = {"name": "MT32_CONTROL.ROM", "md5": "c" * 32, "sha1": ["a" * 40, "b" * 40]} + truth = {"name": "MT32_CONTROL.ROM", "md5": "c" * 32, "sha1": ["a" * 40, "b" * 40], + "validation": ["sha1"]} scraped = {"name": "MT32_CONTROL.ROM", "md5": "d" * 32, "sha1": "b" * 40} self.assertIsNone(_hash_mismatch(truth, scraped)) self.assertEqual( @@ -189,6 +192,40 @@ class EveryRevisionACoreAcceptsStays(unittest.TestCase): +class DiffAndExportShareOneMismatchRule(unittest.TestCase): + """diff_truth let one shared value settle every field and read no + sha256, while the export corrected each field on its own: the diff + stayed silent on hashes the export rewrote.""" + + CASES = ( + ({"sha1": "a" * 40, "md5": "1" * 32}, {"sha1": "a" * 40, "md5": "2" * 32}), + ({"sha256": "a" * 64}, {"sha256": "b" * 64}), + ({"sha1": "a" * 40, "md5": "1" * 32}, {"sha1": "a" * 40, "md5": "1" * 32}), + ) + + def test_the_diff_names_what_the_export_corrects(self): + from exporter.baseline import build_native_model # noqa: PLC0415 + from truth import diff_platform_truth # noqa: PLC0415 + + for t_hashes, p_hashes in self.CASES: + with self.subTest(truth=t_hashes, platform=p_hashes): + truth = {"systems": {"s": {"files": [ + {"name": "f.bin", "required": True, "_cores": ["c"], **t_hashes}]}}} + scraped = {"systems": {"s": {"files": [ + {"name": "f.bin", "destination": "f.bin", "required": True, **p_hashes}]}}} + diff = diff_platform_truth(truth, scraped) + reported = { + m["hash_type"] + for system in diff.get("divergences", {}).values() + for m in system.get("hash_mismatch", []) + } + _, report = build_native_model(truth, scraped) + corrected = {item.split()[-1] for item in report.hashes_corrected} + # One record per entry, naming the first field the export corrects. + self.assertEqual(bool(reported), bool(corrected)) + self.assertLessEqual(reported, corrected) + + def _filed(truth: dict, name: str) -> list[str]: """The systems an entry of that name was filed under.""" return sorted( @@ -454,5 +491,58 @@ class TheModelIsAFunctionOfItsInputs(unittest.TestCase): ) +class DatabaseSizeRefutesASameNamedDeclaration(unittest.TestCase): + """Batocera's BK system declares BASIC10.ROM by md5 alone. clk's Oric + basic10.rom, 16 KB with its own system:, was filed under the BK and + added to its list: the 24576 bytes the database holds for that md5 + were never compared.""" + + def test_the_size_behind_a_declared_hash_counts(self): + bk_md5, oric_md5 = "3" * 32, "e" * 32 + db = {"files": {"1" * 40: {"sha1": "1" * 40, "md5": bk_md5, "size": 24576}}, + "indexes": {"by_md5": {bk_md5: "1" * 40}}} + profiles = {"clk": {"emulator": "clk", "type": "libretro", "systems": ["oric"], + "files": [{"name": "basic10.rom", "path": "Oric/basic10.rom", + "system": "oric", "size": 16384, "md5": oric_md5}]}} + config = {"cores": ["clk"], "systems": {"elektronika-bk": {"files": [ + {"name": "BASIC10.ROM", "md5": bk_md5}]}}} + truth = generate_platform_truth("p", config, {}, profiles, db=db) + self.assertNotIn("basic10.rom", { + f["name"] for f in truth["systems"].get("elektronika-bk", {}).get("files", [])}) + + +class RegionFollowsEveryReader(unittest.TestCase): + """The first core to tag a file set its region in the truth: the ST-V + BIOS kept beetle_saturn's Americas and lost mednafen_stv's Korea, and + psxonpsp660.bin read japan although beetle_psx declares it untagged. + The pack's region index takes the union and voids it on an untagged + reader, and the truth now says the same.""" + + @staticmethod + def _profile(name: str, region: list[str] | None) -> dict: + entry = {"name": "bios.bin", "system": "sega-stv", "md5": "a" * 32} + if region is not None: + entry["region"] = region + return {"emulator": name, "type": "libretro", "systems": ["sega-stv"], + "files": [entry]} + + def _region(self, profiles: dict) -> object: + config = {"cores": sorted(profiles), "systems": {"sega-stv": {"files": []}}} + truth = generate_platform_truth("p", config, {}, profiles) + (entry,) = truth["systems"]["sega-stv"]["files"] + self.assertNotIn("_region_void", entry) + return entry.get("region") + + def test_tags_of_every_reader_are_kept(self): + region = self._region({"a": self._profile("a", ["north-america"]), + "b": self._profile("b", ["south-korea", "north-america"])}) + self.assertEqual(region, ["north-america", "south-korea"]) + + def test_an_untagged_reader_leaves_no_region(self): + for order in (("a", "b"), ("b", "a")): + profiles = {"a": self._profile("a", ["japan"]), "b": self._profile("b", None)} + self.assertIsNone(self._region({k: profiles[k] for k in order})) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_validate_pr.py b/tests/test_validate_pr.py index b5acf0c59..8cbded39b 100644 --- a/tests/test_validate_pr.py +++ b/tests/test_validate_pr.py @@ -329,7 +329,8 @@ class Reporting(_Fixture): class PlatformHashLoading(_Fixture): def test_missing_directory_yields_empty_sets(self): known = validate_pr.load_platform_hashes(str(self.tmp / "absent")) - self.assertEqual(known, {"sha1": set(), "md5": set(), "names": set(), "folders": {}, "sizes": {}}) + self.assertEqual(known, {"sha1": set(), "md5": set(), "sha256": set(), "crc32": set(), + "names": set(), "folders": {}, "sizes": {}, "entries": {}}) def test_declared_hashes_and_names_are_collected(self): platforms = self.tmp / "platforms" @@ -352,6 +353,77 @@ class PlatformHashLoading(_Fixture): self.assertIn("boot.bin", known["names"]) +class ThePlatformsOwnCheckDecides(_Fixture): + """Batocera pins a ROM inside adam_fdc.zip, Recalbox hashes a ZIP's + members, RetroDECK declares a sha256: the container's sha1 and md5 are + in none of those lists, and a file verify counts OK was reported as an + unknown variant to be filed under .variants/.""" + + def _platforms(self, registry: str, systems: str) -> dict: + platforms = self.tmp / "platforms" + platforms.mkdir() + (platforms / "_registry.yml").write_text( + f"platforms:\n p:\n config: p.yml\n status: active\n{registry}", + encoding="utf-8") + (platforms / "p.yml").write_text(f"platform: P\nsystems:\n s:\n files:\n{systems}", + encoding="utf-8") + return validate_pr.load_platform_hashes(str(platforms)) + + def _zip(self) -> str: + import zipfile # noqa: PLC0415 + + path = self.tmp / "contrib" / "set.zip" + path.parent.mkdir() + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("rom.u10", b"ROM") + return "contrib/set.zip" + + def _verdict(self, path: str, known: dict) -> str: + return validate_pr.validate_file(path, None, known).to_markdown() + + def test_a_member_pinned_by_zipped_file(self): + import hashlib # noqa: PLC0415 + + md5 = hashlib.md5(b"ROM").hexdigest() + known = self._platforms( + " verification_mode: md5\n", + f" - {{name: set.zip, md5: {md5}, zipped_file: rom.u10}}\n") + markdown = self._verdict(self._zip(), known) + self.assertIn("Accepted by the platform's own check: p", markdown) + self.assertNotIn(".variants/", markdown) + + def test_a_member_digest(self): + from hashing import md5_composite # noqa: PLC0415 + + path = self._zip() + known = self._platforms( + " verification_mode: md5\n zip_digest: composite\n", + f" - {{name: set.zip, md5: {md5_composite(path)}}}\n") + self.assertIn("Accepted by the platform's own check: p", self._verdict(path, known)) + + def test_a_sha256(self): + import hashlib # noqa: PLC0415 + + path = self._zip() + digest = hashlib.sha256((self.tmp / path).read_bytes()).hexdigest() + known = self._platforms(" verification_mode: md5\n", + f" - {{name: set.zip, sha256: {digest}}}\n") + self.assertIn("Hash matches known platform requirement", self._verdict(path, known)) + + def test_an_identical_copy_names_the_folder(self): + db = {"files": {"m" * 40: {"path": "bios/Arcade/MAME/set.zip"}, + "c" * 40: {"path": "bios/Coleco/Adam/set.zip"}}, + "indexes": {"by_name": {"set.zip": ["m" * 40, "c" * 40]}}} + self.assertEqual(validate_pr._variant_home(db, "set.zip", "c" * 40), "bios/Coleco/Adam/") + self.assertEqual(validate_pr._variant_home(db, "set.zip"), "bios/Arcade/MAME/") + + def test_another_member_is_still_a_variant(self): + known = self._platforms( + " verification_mode: md5\n", + f" - {{name: set.zip, md5: {'0' * 32}, zipped_file: rom.u10}}\n") + self.assertIn("may be a variant", self._verdict(self._zip(), known)) + + class FlagsAreAppliedOrRefused(unittest.TestCase): """Files named beside --changed were dropped for the git diff and the run said "No changed BIOS files detected" about files it never read; --json