From 2c76cc7df9acebfe8271e7f19c22843978a5a49d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:39:46 +0100 Subject: [PATCH] feat: read a search list as one need A core that loads the first of several names (yabause: saturn_bios.bin, sega_101.bin, mpr-17933.bin) needs one of them. Profiles wrote this as required on the first name and optional on the others, so verify and the gap report called saturn_bios.bin a missing required file while sega_101.bin was held, and a set held only through a name marked optional read as a need met by an optional file. Model: an explicit group on the file entry, mirroring the platform field. priority was not reused: it is a rank, read by slot.py, region.py and truth.py inside a (system, region) slot, and that slot is not the set. px68k-sdl ranks its IPL list and its font list each from 1 in one system, pom2 ranks one list per machine model, DuckStation ranks 105 images across every region and takes any of them. A rank cannot say where one list ends. priority stays the order inside a group. Readers, through alternatives.settle_sets: - verify --emulator and the emulator pack: a held member fills the set, the others leave the report, a set nobody holds is one gap under its first member naming the others. - find_undeclared_files (verify platform, core complement of packs and manifests): same rule, a platform declaration or launcher counts as answering the set. - cross_reference: excused members are set aside as other versions. - slot.py: ranks compare members of one set only. - archive members: a required set inside an archive is met by one. - truth: a member gives no requirement of its own, one_of and one_of_required carry the set, and the exporters and the diff leave such a line's required as the platform wrote it. - validate_schemas: members of a group share required. --- schemas/emulator.schema.json | 9 +- scripts/alternatives.py | 87 +++++++++++ scripts/common.py | 40 ++++- scripts/cross_reference.py | 29 +++- scripts/exporter/baseline.py | 7 + scripts/exporter/recalbox_exporter.py | 7 +- scripts/generate_pack.py | 33 +++- scripts/slot.py | 15 +- scripts/truth.py | 35 ++++- scripts/validate_schemas.py | 25 +++ scripts/verify.py | 134 +++++++++++++--- tests/test_e2e.py | 214 ++++++++++++++++++++++++++ wiki/profiling.md | 5 +- 13 files changed, 599 insertions(+), 41 deletions(-) create mode 100644 scripts/alternatives.py diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index dab1a1d7a..b6ac0b212 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -674,10 +674,17 @@ ] }, "priority": { - "description": "Preference order among the BIOS a system accepts, lowest first. DuckStation's FindBIOSImageInDirectory keeps the image whose priority is lower. A search list the code walks in order maps onto it as 1, 2, 3.", + "description": "Preference order among the BIOS a system accepts, lowest first. DuckStation's FindBIOSImageInDirectory keeps the image whose priority is lower. A search list the code walks in order maps onto it as 1, 2, 3. The order only: which entries stand in for one another is group.", "type": "integer", "minimum": 0 }, + "group": { + "description": "Entries of one profile sharing a group are one need: the code loads one of them (the first it finds, in priority order where the source states one) and holding any member fills it. required states the set's requirement and is the same on every member. A list puts a file in several sets, a fallback two machine models share.", + "oneOf": [ + {"type": "string", "minLength": 1}, + {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "minLength": 1}} + ] + }, "region_check": { "type": "boolean" }, diff --git a/scripts/alternatives.py b/scripts/alternatives.py new file mode 100644 index 000000000..ef15290f5 --- /dev/null +++ b/scripts/alternatives.py @@ -0,0 +1,87 @@ +"""Entries a core accepts in place of one another. + +A profile writes ``group:`` on the entries of one search list. Yabause tries +saturn_bios.bin, sega_101.bin, then mpr-17933.bin and loads the first it +finds (libretro.c:1035-1049): it needs one of them and none in particular. +``required`` on a member states the set's requirement, and ``priority`` keeps +the order the code reads them in. + +Holding one member fills the set, so the other members are not missing. A +set nobody holds is one gap, reported under its first member, never one gap +per member. Every reader settles sets through ``settle_sets`` so that verify, +the gap report and the packs agree on what is missing. +""" + +from __future__ import annotations + +from collections.abc import Hashable, Iterable, Sequence + +_UNRANKED = 1 << 30 + + +def groups_of(entry: dict) -> tuple[str, ...]: + """The sets an entry belongs to: one name, or a list for a file two + machine models both fall back on.""" + value = entry.get("group") + if not value: + return () + values = value if isinstance(value, list) else [value] + return tuple(str(v) for v in values if v) + + +def set_keys(owner: Hashable, entry: dict) -> tuple[tuple, ...]: + """The keys of an entry's sets, scoped to its owner (a profile, or a + profile under one build).""" + return tuple((owner, group) for group in groups_of(entry)) + + +def slot_set_keys(owner: Hashable, entries: Iterable[dict]) -> tuple[tuple, ...]: + """The sets the entries on one destination belong to. A destination has + one outcome however many versions it lists, so its versions' sets are + settled together.""" + keys: list[tuple] = [] + for entry in entries: + keys.extend(k for k in set_keys(owner, entry) if k not in keys) + return tuple(keys) + + +def _rank(entry: dict, position: int) -> tuple[int, int]: + priority = entry.get("priority") + return (priority if isinstance(priority, int) else _UNRANKED, position) + + +def settle_sets( + members: Sequence[tuple[tuple[tuple, ...], dict, bool]], +) -> tuple[set[int], dict[int, list[str]]]: + """Which unheld members are excused, and which one stands for its set. + + ``members`` lists ``(set keys, entry, fills)`` in the order the profile + declares them, where ``fills`` says the member is held (or answered by a + declaration that reports it). Returns the positions to leave out of a + report and, for each set no member fills, the position kept as its one + gap with the names of the others. A member fills or not on its own: a + held one is never excused, and an unheld one stays reported when it + stands for at least one unfilled set. + """ + by_set: dict[tuple, list[int]] = {} + for position, (keys, _entry, _fills) in enumerate(members): + for key in keys: + by_set.setdefault(key, []).append(position) + + standing: dict[int, list[str]] = {} + for positions in by_set.values(): + if any(members[p][2] for p in positions): + continue + first = min(positions, key=lambda p: _rank(members[p][1], p)) + names = standing.setdefault(first, []) + for p in positions: + name = str(members[p][1].get("name", "")) + if p != first and name not in names: + names.append(name) + + excused = { + position + for position, (keys, _entry, fills) in enumerate(members) + if keys and not fills and position not in standing + } + return excused, standing diff --git a/scripts/common.py b/scripts/common.py index 968320769..2cef0d5d9 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -1267,7 +1267,39 @@ _archive_index: dict[int, tuple[list[dict], int, dict[str | None, list[dict]]]] def _member(f: dict) -> dict: - return {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))} + member = {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))} + if f.get("group"): + # One of the members sharing it is enough (alternatives.py). + member["group"] = f["group"] + return member + + +def _required_met(members: list[dict], held: Callable[[dict], bool]) -> tuple[int, bool]: + """How many requirements an archive meets, and whether it meets all. + + A required member is one requirement, and so is a required set of + alternatives, met when one of its members is held: Odin reads the + first of three Hikaru boot ROMs it finds in hikaru.zip. + """ + from alternatives import groups_of + + met = 0 + complete = True + sets: dict[str, bool] = {} + for m in members: + if not m["required"]: + continue + names = groups_of(m) + if names: + for name in names: + sets[name] = sets.get(name, False) or held(m) + continue + if held(m): + met += 1 + else: + complete = False + met += sum(sets.values()) + return met, complete and all(sets.values()) def archive_members(archive: str, files: list[dict]) -> list[dict]: @@ -1316,7 +1348,7 @@ def holds_required_members(path: str, members: list[dict]) -> bool: """Whether an archive holds every member a profile requires inside it.""" crcs = zip_member_crcs(path) names = zip_member_names(path) - return all(_member_held(m, crcs, names) for m in members if m["required"]) + return _required_met(members, lambda m: _member_held(m, crcs, names))[1] def contents_members(file_entry: dict) -> list[dict]: @@ -1360,8 +1392,8 @@ def _holding_members(paths: list[str], members: list[dict]) -> list[str]: def held(path: str) -> tuple[int, int]: crcs = zip_member_crcs(path) names = zip_member_names(path) - found = [m for m in members if _member_held(m, crcs, names)] - return sum(1 for m in found if m["required"]), len(found) + found = {id(m) for m in members if _member_held(m, crcs, names)} + return _required_met(members, lambda m: id(m) in found)[0], len(found) scores = {path: held(path) for path in paths} best = max(scores.values(), default=(0, 0)) diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index af7525a77..8866b3420 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -23,6 +23,7 @@ from collections import Counter from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) +from alternatives import set_keys, settle_sets from common import ( platform_arg, ProfileSelectionError, @@ -797,6 +798,9 @@ def _cross_reference_profile( # the profile's entries in their order, so a position names one entry. position = {id(f): i for view, _ in layouts for i, f in enumerate(view.get("files", []))} fates: dict[int, str] = {} + # Grouped entries met, as (set keys, entry, fills, the list holding + # its row, the row): settled once the profile is read. + set_records: list[tuple[tuple, dict, bool, list | None, dict | None]] = [] def fate(f: dict, what: str) -> None: key = position[id(f)] @@ -834,15 +838,18 @@ def _cross_reference_profile( # Collect unsourceable files separately (documented, not a gap) unsourceable_reason = f.get("unsourceable", "") + keys = set_keys((emu_name, is_standalone), f) if unsourceable_reason: seen_files.add(seen_key) fate(f, "unsourceable") - unsourceable_list.append({ + row = { "name": fname, "required": f.get("required", False), "reason": unsourceable_reason, "source_ref": f.get("source_ref", ""), - }) + } + unsourceable_list.append(row) + set_records.append((keys, f, False, unsourceable_list, row)) continue if outside_gap_scope(f, is_standalone): @@ -863,6 +870,7 @@ def _cross_reference_profile( if in_platform: seen_files.add(seen_key) fate(f, "declared") + set_records.append((keys, f, True, None, None)) covered.append({ "name": fname, "path": effective_path, @@ -955,6 +963,21 @@ def _cross_reference_profile( (emu_name, is_standalone, f, probe, entry) ) gaps.append(entry) + set_records.append((keys, f, in_repo, gaps, entry)) + + # One need per set: a member another one fills is not a gap, and a set + # nobody holds is one gap under its first member (alternatives.py), as + # verify and the packs count it. + grouped = [r for r in set_records if r[0]] + excused, standing = settle_sets([(keys, f, fills) for keys, f, fills, _l, _r in grouped]) + for at, (_keys, member, _fills, rows, set_row) in enumerate(grouped): + if set_row is None or rows is None: + continue + if at in standing: + set_row["alternatives"] = standing[at] + if at in excused: + rows.remove(set_row) + fates[position[id(member)]] = "other_versions" for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]): gaps.append(ag) @@ -1333,6 +1356,8 @@ def print_report( fc = g.get("archive_file_count", 0) rc = g.get("archive_required_count", 0) archive_info = f" ({fc} files, {rc} required)" + if g.get("alternatives") and not g.get("in_repo"): + archive_info += f" (or {', '.join(g['alternatives'])})" print(f" {req} {g['name']} [{src}]{archive_info}{note}") total_gaps += gaps diff --git a/scripts/exporter/baseline.py b/scripts/exporter/baseline.py index af09d300f..66f5a3b82 100644 --- a/scripts/exporter/baseline.py +++ b/scripts/exporter/baseline.py @@ -264,6 +264,8 @@ class NativeFile: return bool(self.truth["required"]) if self.platform is not None and self.platform.get("required") is not None: return bool(self.platform["required"]) + if self.truth is not None and self.truth.get("one_of_required") is not None: + return bool(self.truth["one_of_required"]) return True @property @@ -645,10 +647,15 @@ def build_native_model( matched.filled.append(field_name) t_req = truth_entry.get("required") p_req = (matched.platform or {}).get("required") + # A file a core takes as one of a set has no requirement of + # its own: a platform line cannot say "one of these", so + # truth leaves its value as written rather than claim + # every member optional or every member needed. if ( t_req is not None and p_req is not None and bool(t_req) != bool(p_req) + and not (truth_entry.get("one_of_required") and not t_req) ): matched.corrections.append("required") report.required_corrected.append( diff --git a/scripts/exporter/recalbox_exporter.py b/scripts/exporter/recalbox_exporter.py index 161c9a9bb..640658bd8 100644 --- a/scripts/exporter/recalbox_exporter.py +++ b/scripts/exporter/recalbox_exporter.py @@ -189,16 +189,21 @@ class Exporter(BaseExporter): } - {""} readers: set[str] = set() requirers: set[str] = set() + set_readers: set[str] = set() for entry in entries: cores = set(entry.get("_cores") or ()) & named readers |= cores + if entry.get("one_of_required"): + set_readers |= cores & set(entry.get("one_of") or ()) required_by = entry.get("_required_by") if required_by is None: if entry.get("required") is not False: requirers |= cores else: requirers |= cores & set(required_by) - if not readers: + # A core that needs one file of a set needs none of them on its own, + # which a mandatory flag cannot say: the line keeps its value. + if not readers or (not requirers and set_readers): if fe.platform is not None and fe.platform.get("required") is not None: return bool(fe.platform["required"]) return fe.required diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 75bba4923..17adf5979 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -36,6 +36,7 @@ import zipfile from pathlib import Path, PurePosixPath sys.path.insert(0, os.path.dirname(__file__)) +from alternatives import settle_sets, slot_set_keys from common import resolve_platform_cores as _platform_cores from common import ( installed_data_refs, @@ -1838,6 +1839,10 @@ def generate_emulator_pack( continue slots.setdefault(dest, []).append(fe) + # What each destination came to, settled by set once the loop + # is over: a member another member fills is not missing, and a + # set nobody holds is reported once (alternatives.py). + outcomes: list[tuple[list[dict], bool, Callable[[], None] | None]] = [] for dest, alternatives in slots.items(): # Name order, first file wins, case folded: verify replays # the rule and reports a later profile's file as missing. @@ -1923,17 +1928,25 @@ def generate_emulator_pack( held, packed_at.get(dest.lower()) ): # The file already at this path, asked for twice. + outcomes.append((alternatives, True, None)) continue - report_missing(alternatives[0]["name"], dest) + outcomes.append((alternatives, False, functools.partial( + report_missing, alternatives[0]["name"], dest + ))) continue if chosen is None: if refusal: rejected_files.append(refusal) + outcomes.append((alternatives, False, None)) elif all(fe.get("unsourceable") for fe in alternatives): - unsourceable_files.append(alternatives[0]["name"]) + outcomes.append((alternatives, False, functools.partial( + unsourceable_files.append, alternatives[0]["name"] + ))) else: - report_missing(alternatives[0]["name"], dest) + outcomes.append((alternatives, False, functools.partial( + report_missing, alternatives[0]["name"], dest + ))) continue fe, local_path, how = chosen @@ -1954,8 +1967,11 @@ def generate_emulator_pack( total_files += 1 if record is not None: record.files.append(dest) + outcomes.append((alternatives, True, None)) else: - missing_files.append(fe["name"]) + outcomes.append((alternatives, False, functools.partial( + missing_files.append, fe["name"] + ))) finally: if os.path.exists(tmp_path): os.unlink(tmp_path) @@ -1969,6 +1985,7 @@ def generate_emulator_pack( # (but allow same file to be packed under different destinations, # e.g., IPL.bin in GC/USA/ and GC/EUR/ from same source) dedup_key_hash = f"{os.path.realpath(local_path)}:{dest}" + outcomes.append((alternatives, True, None)) if dedup_key_hash in seen_hashes: continue seen_hashes.add(dedup_key_hash) @@ -1985,6 +2002,14 @@ def generate_emulator_pack( if record is not None: record.files.append(dest) + excused, _standing = settle_sets([ + (slot_set_keys(emu_name, entries), entries[0], fills) + for entries, fills, _report in outcomes + ]) + for position, (_entries, _fills, report) in enumerate(outcomes): + if report is not None and position not in excused: + report() + # The trees after the files, as in a platform pack: a dump the # profile declares wins over the copy a tree bundles at the same # path (Dolphin's real dsp_rom.bin over Sys/GC's free one). diff --git a/scripts/slot.py b/scripts/slot.py index d5df6ffe0..8ab85b58f 100644 --- a/scripts/slot.py +++ b/scripts/slot.py @@ -29,6 +29,8 @@ rank: the second core reads it by its own rule, which no rank expresses. from __future__ import annotations +from alternatives import set_keys + BIOS_CATEGORY = "bios" @@ -65,9 +67,11 @@ def build_slot_index(profiles: dict) -> dict[str, dict]: entry = index.setdefault( key, {"rank": None, "conflict": False, "regions": set(), - "emulators": [], "ranked_by": set(), "unranked_by": set()}, + "emulators": [], "ranked_by": set(), "unranked_by": set(), + "sets": set()}, ) entry["regions"] |= {str(r) for r in (f.get("region") or [])} + entry["sets"] |= set(set_keys(emu_name, f)) rank = f.get("priority") entry["unranked_by" if rank is None else "ranked_by"].add(emu_name) if rank is not None: @@ -123,7 +127,7 @@ def resolve_slot_drops( # A slot is a system AND a declared region: the Japanese and American # PlayStation BIOS are not alternatives to each other, and comparing their # ranks across regions would pick one territory's file for another's. - tiers: dict[tuple[str, tuple[str, ...]], list[tuple[str, int | None]]] = {} + tiers: dict[tuple[str, tuple[str, ...], tuple], list[tuple[str, int | None]]] = {} for group_id, members in groups.items(): for destination, name in members: if held is not None and destination not in held: @@ -132,11 +136,14 @@ def resolve_slot_drops( if entry is None: keep.add(destination) continue + # A rank orders the members of one set: px68k-sdl ranks its IPL + # and its font ROM lists each from 1, and the two never compete. tier = tuple(sorted(entry["regions"])) + sets = tuple(sorted(entry.get("sets") or ())) rank = None if entry.get("conflict") else entry["rank"] - tiers.setdefault((group_id, tier), []).append((destination, rank)) + tiers.setdefault((group_id, tier, sets), []).append((destination, rank)) - for (group_id, tier), candidates in tiers.items(): + for (group_id, tier, _sets), candidates in tiers.items(): if len(candidates) < 2: keep.update(dest for dest, _p in candidates) continue diff --git a/scripts/truth.py b/scripts/truth.py index d463368ac..fd466b4ca 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -18,6 +18,7 @@ from common import ( resolve_platform_cores, standalone_set as platform_standalone_set, ) +from alternatives import groups_of from hashing import compute_hashes from validation import ( filter_files_by_mode, @@ -276,11 +277,15 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None: existing["_source_refs"] = existing.get("_source_refs", set()) | {sr_key} else: existing.setdefault("_source_refs", set()) - if file_entry.get("required"): + if groups_of(file_entry): + _fold_set_member(existing, file_entry, emu_name) + elif file_entry.get("required"): existing["required"] = True # Which cores need it: `required` above is the union, true when # any core needs it, and a package of one core must not read it. existing["_required_by"] = existing.get("_required_by", set()) | {emu_name} + elif file_entry.get("required") is not None: + existing.setdefault("required", file_entry["required"]) _merge_hashes(existing, file_entry, emu_name, revision) # Merge non-hash data fields if existing lacks them. # A core that creates an entry without size/path/validation may be @@ -292,6 +297,21 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None: _merge_region(existing, file_entry.get("region")) +def _fold_set_member(entry: dict, file_entry: dict, emu_name: str) -> None: + """A core that takes this file as one of a set needs the set, not it. + + Yabause loads the first of saturn_bios.bin, sega_101.bin and + mpr-17933.bin it finds. None of the three is required on its own, so + the core stays out of `required` and `_required_by`, and `_one_of` + names it with `one_of_required` saying whether its set is required. + """ + entry["_one_of"] = entry.get("_one_of", set()) | {emu_name} + if file_entry.get("required", True): + entry["one_of_required"] = True + else: + entry.setdefault("one_of_required", False) + + def _merge_region(existing: dict, theirs: str | list | None) -> None: """The regions every reader selects the file for, the pack's own rule. @@ -379,7 +399,8 @@ def _merge_priority(existing: dict, theirs: int | None) -> None: def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict: entry: dict = {"name": file_entry["name"]} - if file_entry.get("required") is not None: + grouped = bool(groups_of(file_entry)) + if file_entry.get("required") is not None and not grouped: entry["required"] = file_entry["required"] for field in _COPIED_FIELDS: val = file_entry.get(field) @@ -396,7 +417,9 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict: if not file_entry.get("region"): entry["_region_void"] = True entry["_cores"] = {emu_name} - entry["_required_by"] = {emu_name} if file_entry.get("required") else set() + entry["_required_by"] = {emu_name} if file_entry.get("required") and not grouped else set() + if grouped: + _fold_set_member(entry, file_entry, emu_name) entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set() entry["_brought_by"] = _brought(file_entry, emu_name) sr = file_entry.get("source_ref") @@ -848,6 +871,9 @@ def generate_platform_truth( fe.pop("_region_void", None) fe["_cores"] = sorted(fe.get("_cores", set())) fe["_required_by"] = sorted(fe.get("_required_by", set())) + one_of = fe.pop("_one_of", None) + if one_of: + fe["one_of"] = sorted(one_of) fe["_named_by"] = sorted(fe.get("_named_by", set())) fe["_brought_by"] = sorted(fe.get("_brought_by", set())) if not fe["_brought_by"]: @@ -950,6 +976,9 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict: hash_mismatch.append(mismatch) t_req = t_entry.get("required") s_req = s_entry.get("required") + if t_entry.get("one_of_required") and not t_req: + # Needed as one of a set: no value of its own to compare. + t_req = None if t_req is not None and s_req is not None and t_req != s_req: required_mismatch.append( { diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index db84dbc06..0ff95263e 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -86,6 +86,7 @@ def _validate_yaml_directory( out.extend(_save_dir_without_library_name(path, data)) out.extend(_systems_outside_profile(path, data)) out.extend(_doubled_paths(path)) + out.extend(_group_requirement_split(path, data)) return out @@ -122,6 +123,30 @@ def _systems_outside_profile(path: Path, document: object) -> list[str]: return out +def _group_requirement_split(path: Path, document: object) -> list[str]: + """A group whose members disagree on required. + + The members of a group are one need, so required states the set's and + reads the same on each. A true on the first name only wrote the search + order into required, and the other names read as optional files. + """ + from alternatives import groups_of + + if not isinstance(document, dict): + return [] + seen: dict[str, set[bool]] = {} + for entry in document.get("files") or []: + if not isinstance(entry, dict): + continue + for group in groups_of(entry): + seen.setdefault(group, set()).add(bool(entry.get("required", True))) + return [ + f"{path.relative_to(ROOT)}: group {group} mixes required true and false" + for group, values in sorted(seen.items()) + if len(values) > 1 + ] + + def _checks_without_values(path: Path, document: object) -> list[str]: """A check an entry declares with nothing to compare against. diff --git a/scripts/verify.py b/scripts/verify.py index 437077555..29c64e108 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -37,6 +37,7 @@ import slots sys.path.insert(0, os.path.dirname(__file__)) from hashing import parse_md5_list +from alternatives import set_keys, settle_sets, slot_set_keys from common import ( _foreign_tree, _owner_tokens, @@ -534,6 +535,9 @@ def find_undeclared_files( config, profiles, relevant, db, declared_destinations(config) ) launcher_candidates: list[tuple[str, bool, dict, dict]] = [] + # Each grouped requirement met, as (set keys, entry, fills, (reported + # entry, the entry it is a claim on)): settled once every core is read. + set_records: list[tuple[tuple, dict, bool, tuple[dict, dict | None] | None]] = [] undeclared = [] seen_files: set[tuple] = set() # Key -> the entry reported for it. A later profile reading the same @@ -606,6 +610,9 @@ def find_undeclared_files( ) if verdict == "settled": seen_files.add(seen_key) + # Answered by the platform's own declaration, which its + # report covers: the set it belongs to is spoken for. + set_records.append((set_keys((emu_name, is_standalone), f), f, True, None)) continue if verdict == "skip": continue @@ -761,12 +768,16 @@ def find_undeclared_files( entry["members"] = members if launcher_layer and not save_root: launcher_candidates.append((emu_name, is_standalone, f, entry)) + keys = set_keys((emu_name, is_standalone), f) if duplicate: first.setdefault("other_claims", []).append(entry) + set_records.append((keys, f, in_repo, (entry, first))) continue reported[seen_key] = entry undeclared.append(entry) + set_records.append((keys, f, in_repo, (entry, None))) + settled: set[int] = set() if launcher_layer: added, settled = launcher_layer.settle( launcher_candidates, lambda report: held_record(report, db) @@ -774,12 +785,63 @@ def find_undeclared_files( undeclared = [u for u in undeclared if id(u) not in settled] undeclared.extend(added) + undeclared = _settle_core_sets(undeclared, set_records, settled) + for entry in sorted(archive_entries.values(), key=lambda e: e["name"]): undeclared.append(entry) return undeclared +def _settle_core_sets( + undeclared: list[dict], + records: list[tuple[tuple, dict, bool, tuple[dict, dict | None] | None]], + launcher_settled: set[int], +) -> list[dict]: + """The core complement with each set of alternatives settled. + + A member the collection lacks is dropped when another member is held, + declared by the platform or placed by its launcher, and a set nobody + holds keeps one entry, its first member, naming the others. A claim + another profile shares stays: its own core may need that very file. + """ + grouped = [r for r in records if r[0]] + if not grouped: + return undeclared + excused, standing = settle_sets([ + (keys, f, fills or (ref is not None and id(ref[0]) in launcher_settled)) + for keys, f, fills, ref in grouped + ]) + dropped: set[int] = set() + for position, (_keys, _f, _fills, ref) in enumerate(grouped): + if ref is None: + continue + entry, host = ref + if position in standing: + entry["alternatives"] = standing[position] + if position not in excused: + continue + if host is not None: + host["other_claims"] = [c for c in host.get("other_claims", []) if c is not entry] + if not host["other_claims"]: + del host["other_claims"] + else: + dropped.add(id(entry)) + out: list[dict] = [] + for entry in undeclared: + if id(entry) not in dropped: + out.append(entry) + continue + # Another core reading the same requirement still needs it. + claims = entry.get("other_claims") or [] + if claims: + promoted = dict(claims[0]) + if claims[1:]: + promoted["other_claims"] = claims[1:] + out.append(promoted) + return out + + def find_exclusion_notes( config: dict, emulators_dir: str, @@ -1802,6 +1864,8 @@ def _print_undeclared_entry( name_label = f"{name} ({arc_count} file{'s' if arc_count != 1 else ''})" else: name_label = name + if u.get("alternatives") and not u.get("in_repo"): + name_label += f" (or {', '.join(u['alternatives'])})" print(f" {prefix}: {u['emulator']} needs {name_label}") checks = u.get("checks", []) if checks: @@ -2171,6 +2235,9 @@ def verify_emulator( return local_path, status details = [] + # Members of a set another member fills, or that the set's first member + # stands for: not missing, and out of every count. + set_members: list[dict] = [] file_status: dict[str, str] = {} file_severity: dict[str, str] = {} dest_to_name: dict[str, str] = {} @@ -2397,7 +2464,9 @@ def verify_emulator( # profile on one destination are the releases the code accepts # there (a shareware and a retail HOG, each GRP in a size table): # holding one of them fills the slot, and the others are not gaps. - emu_results: dict[str, list[tuple[dict, bool, bool]]] = {} + emu_results: dict[str, list[tuple[dict, bool, bool, dict]]] = {} + placed_results: list[tuple[str, dict, bool, bool, list[dict]]] = [] + fallback_reason = "declared hash contradicted by the local file" for file_entry in files: # Skip archive members (verified as archive units above) @@ -2497,13 +2566,13 @@ def verify_emulator( # The slot the emulator pack places this entry in: standalone_path # under --standalone, the sanitised path. dest = _resolve_destination(file_entry, standalone) or name - emu_results.setdefault(dest, []).append((result, required, hle)) + emu_results.setdefault(dest, []).append((result, required, hle, file_entry)) for dest, alternatives in emu_results.items(): # What the emulator pack places here: an accepted file, else a # file whose hash the profile contradicts. A refused copy is held # but never packed. - fallback_reason = "declared hash contradicted by the local file" + slot_entries = [a[3] for a in alternatives] if len(alternatives) > 1: # Reduced in the order generate_emulator_pack chooses, so the # destination's status is the one the pack reports: a slot it @@ -2556,27 +2625,49 @@ def verify_emulator( elif not block and packed is not None: occupy(dest, packed, emu_name) packed_dests.add(dest) - for result, required, hle in alternatives: + for result, required, hle, file_entry in alternatives: if block_reason and result["status"] != Status.MISSING: result["status"] = Status.MISSING result["reason"] = block_reason result["destination"] = dest - details.append(result) + placed_results.append((dest, result, required, hle, slot_entries)) - # Aggregate by destination (path if available, else name) - dest_to_name[dest] = result["name"] - cur = result["status"] - prev = file_status.get(dest) - if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get( - prev, 0 - ): - file_status[dest] = cur - sev = _emulator_severity(cur, required, hle) - prev_sev = file_severity.get(dest) - if prev_sev is None or _SEVERITY_ORDER.get( - sev, 0 - ) > _SEVERITY_ORDER.get(prev_sev, 0): - file_severity[dest] = sev + # A set the code takes one member of is one need: once a member is + # in the pack the others are not missing, and a set nobody holds is + # one gap under its first member. Settled per profile, before the + # destinations of several profiles are merged. + excused, standing = settle_sets([ + ( + slot_set_keys(emu_name, entries), + entries[0], + result["status"] == Status.OK + or (result["status"] == Status.UNTESTED + and result.get("reason") == fallback_reason), + ) + for _dest, result, _required, _hle, entries in placed_results + ]) + for position, (dest, result, required, hle, _fe) in enumerate(placed_results): + if position in excused and result["status"] == Status.MISSING: + set_members.append(result) + continue + if position in standing: + result["alternatives"] = standing[position] + details.append(result) + + # Aggregate by destination (path if available, else name) + dest_to_name[dest] = result["name"] + cur = result["status"] + prev = file_status.get(dest) + if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get( + prev, 0 + ): + file_status[dest] = cur + sev = _emulator_severity(cur, required, hle) + prev_sev = file_severity.get(dest) + if prev_sev is None or _SEVERITY_ORDER.get( + sev, 0 + ) > _SEVERITY_ORDER.get(prev_sev, 0): + file_severity[dest] = sev occupy_trees(emu_name, profile) @@ -2610,6 +2701,7 @@ def verify_emulator( "severity_counts": counts, "status_counts": status_counts, "details": details, + "set_members": set_members, "data_dir_notices": sorted(set(data_dir_notices)), "ground_truth_coverage": { "with_validation": with_validation, @@ -2715,7 +2807,9 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None: seen.add(d["name"]) req = "required" if d.get("required", True) else "optional" hle = ", HLE available" if d.get("hle_fallback") else "" - print(f" MISSING ({req}{hle}): {d['name']}") + others = d.get("alternatives") + either = f" (or {', '.join(others)})" if others else "" + print(f" MISSING ({req}{hle}): {d['name']}{either}") gt = d.get("ground_truth", []) if gt: if verbose: diff --git a/tests/test_e2e.py b/tests/test_e2e.py index ada7bd5ce..6d727698e 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -6765,5 +6765,219 @@ class StandaloneModeRequiresAStandaloneBuild(unittest.TestCase): self.assertFalse(runs_standalone("dolphin", dual, {"pcsx2"})) + +class OneOfASearchList(unittest.TestCase): + """Entries a core takes the first of are one need (``group:``). + + Yabause tries saturn_bios.bin, sega_101.bin, mpr-17933.bin and loads the + first it finds. Written as required on the first name only, verify and + the gap report called saturn_bios.bin a missing required file while + sega_101.bin was held, and a set held through an optional-looking name + passed for the wrong reason. Synthetic: set ``first`` holds only its + second member, set ``none`` holds nothing. + """ + + def setUp(self): + from common import _emulator_profiles_cache + + _emulator_profiles_cache.clear() + repo = Path(__file__).resolve().parent.parent + (repo / "tmp").mkdir(exist_ok=True) + self.root = Path(tempfile.mkdtemp(dir=repo / "tmp")) + self.addCleanup(shutil.rmtree, self.root, True) + previous = os.getcwd() + os.chdir(self.root) + self.addCleanup(os.chdir, previous) + self.db = {"files": {}, "indexes": {"by_md5": {}, "by_name": {}, + "by_crc32": {}, "by_path_suffix": {}}} + for name in ("second.bin", "plain.bin"): + data = name.encode() * 8 + rel = f"bios/Test/Console/{name}" + Path(rel).parent.mkdir(parents=True, exist_ok=True) + Path(rel).write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + self.db["files"][sha1] = {"path": rel, "name": name, "size": len(data), + "md5": hashlib.md5(data).hexdigest(), "crc32": ""} + self.db["indexes"]["by_name"][name] = [sha1] + Path("emulators").mkdir() + Path("platforms").mkdir() + + def member(name: str, group: str, rank: int) -> dict: + return {"name": name, "system": "console-a", "group": group, + "priority": rank, "required": True} + + self.profile = { + "emulator": "Seq", "type": "libretro", "cores": ["seq"], + "systems": ["console-a"], + "files": [ + member("first.bin", "first", 1), + member("second.bin", "first", 2), + member("third.bin", "first", 3), + member("none_a.bin", "none", 1), + member("none_b.bin", "none", 2), + {"name": "plain.bin", "system": "console-a", "required": True}, + ], + } + Path("emulators/seq.yml").write_text(yaml.safe_dump(self.profile)) + self.config = { + "platform": "Seqs", "verification_mode": "existence", + "base_destination": "system", "cores": ["seq"], + "systems": {"console-a": {"files": []}}, + } + + def _verify(self) -> dict: + return verify_emulator(["seq"], "emulators", self.db, platforms_dir="platforms") + + def test_one_member_held_leaves_no_missing_report_for_its_set(self): + result = self._verify() + reported = {d["name"]: d["status"] for d in result["details"]} + self.assertEqual(reported["second.bin"], "ok") + self.assertNotIn("first.bin", reported) + self.assertNotIn("third.bin", reported) + self.assertEqual( + sorted(d["name"] for d in result["set_members"]), + ["first.bin", "none_b.bin", "third.bin"], + ) + + def test_a_set_none_holds_is_one_gap(self): + result = self._verify() + missing = [d for d in result["details"] if d["status"] == "missing"] + self.assertEqual([d["name"] for d in missing], ["none_a.bin"]) + self.assertEqual(missing[0]["alternatives"], ["none_b.bin"]) + # Required and missing, judged as the core judges: one warning. + self.assertEqual(result["severity_counts"][Severity.WARNING], 1) + self.assertEqual(result["severity_counts"][Severity.INFO], 0) + self.assertEqual(result["status_counts"].get("missing"), 1) + self.assertEqual(result["total_files"], 3) + + def test_a_required_set_held_through_a_later_member_is_satisfied(self): + """second.bin used to read required: false beside a required + first.bin: the set is required and held, so nothing is critical + for it, and the strict exit names only the set nobody holds.""" + from verify import _exit_if_strict_emulator + + result = self._verify() + critical = [d["name"] for d in result["details"] + if d["status"] == "missing" and d["required"]] + self.assertEqual(critical, ["none_a.bin"]) + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()) as err: + _exit_if_strict_emulator(True, result) + self.assertIn("1 required file(s) missing: none_a.bin", err.getvalue()) + + def test_pack_and_verify_agree(self): + from generate_pack import generate_emulator_pack + + result = self._verify() + out = io.StringIO() + with contextlib.redirect_stdout(out): + zip_path = generate_emulator_pack( + ["seq"], "emulators", self.db, "bios", "out", offline=True, + platforms_dir="platforms", + ) + with zipfile.ZipFile(zip_path) as zf: + packed = sorted(n for n in zf.namelist() if n != "README.txt") + self.assertEqual(packed, ["plain.bin", "second.bin"]) + self.assertEqual(result["severity_counts"]["ok"], len(packed)) + lines = out.getvalue().splitlines() + self.assertEqual( + [line.split(": ", 1)[1] for line in lines if line.strip().startswith("MISSING:")], + ["none_a.bin"], + ) + self.assertIn("2 files packed, 1 missing", out.getvalue()) + + def test_core_complement_reports_one_gap_per_set(self): + undeclared = find_undeclared_files( + self.config, "emulators", self.db, {"seq": self.profile} + ) + by_name = {u["name"]: u for u in undeclared} + self.assertEqual(sorted(by_name), ["none_a.bin", "plain.bin", "second.bin"]) + self.assertTrue(by_name["second.bin"]["in_repo"]) + self.assertFalse(by_name["none_a.bin"]["in_repo"]) + self.assertEqual(by_name["none_a.bin"]["alternatives"], ["none_b.bin"]) + + def test_a_platform_declaration_speaks_for_the_set(self): + """The platform declares first.bin and reports it in its own check: + the core's other names are not gaps on top of it.""" + config = {**self.config, "systems": {"console-a": {"files": [ + {"name": "first.bin", "destination": "first.bin", "required": True}, + {"name": "none_b.bin", "destination": "none_b.bin", "required": True}, + ]}}} + undeclared = find_undeclared_files( + config, "emulators", self.db, {"seq": self.profile} + ) + self.assertEqual(sorted(u["name"] for u in undeclared), ["plain.bin", "second.bin"]) + + def test_gap_report_counts_the_set_once(self): + import cross_reference + + index = { + "by_name": self.db["indexes"]["by_name"], + "by_name_lower": {k.lower(): k for k in self.db["indexes"]["by_name"]}, + "by_path_suffix": {}, "by_md5": {}, "by_sha256": {}, "by_crc32": {}, + "data_names": set(), "all_declared": set(), "db_files": self.db["files"], + } + report: dict = {} + cross_reference._cross_reference_profile("seq", self.profile, {}, report, index) + gaps = {g["name"]: g for g in report["seq"]["gap_details"]} + self.assertEqual(sorted(gaps), ["none_a.bin", "plain.bin", "second.bin"]) + self.assertEqual(gaps["none_a.bin"]["alternatives"], ["none_b.bin"]) + self.assertEqual(report["seq"]["set_aside"]["other_versions"], 3) + + def test_truth_names_no_member_required_on_its_own(self): + from truth import _fold_into, _new_truth_entry + + entry = _new_truth_entry(self.profile["files"][0], "seq", None) + self.assertNotIn("required", entry) + self.assertEqual(entry["_required_by"], set()) + self.assertTrue(entry["one_of_required"]) + _fold_into(entry, {"name": "first.bin", "required": True}, "other") + self.assertTrue(entry["required"]) + self.assertEqual(entry["_required_by"], {"other"}) + + def test_ranks_order_one_set_and_never_two(self): + """px68k-sdl ranks its IPL list and its font list each from 1: + --one-per-slot keeps the first of each, not one file of the two.""" + from slot import build_slot_index, resolve_slot_drops + + profile = {"type": "standalone", "files": [ + {"name": "ipl_a.dat", "group": "ipl", "priority": 1}, + {"name": "ipl_b.dat", "group": "ipl", "priority": 2}, + {"name": "font_a.dat", "group": "font", "priority": 1}, + {"name": "font_b.dat", "group": "font", "priority": 2}, + ]} + index = build_slot_index({"x68": profile}) + groups = {"sharp-x68000": [(n, n) for n in + ("ipl_a.dat", "ipl_b.dat", "font_a.dat", "font_b.dat")]} + drops, undecidable = resolve_slot_drops(groups, index) + self.assertEqual(drops, {"ipl_b.dat", "font_b.dat"}) + self.assertEqual(undecidable, []) + + def test_an_archive_needs_one_member_of_a_set(self): + import zlib + + from common import holds_required_members + + path = self.root / "set.zip" + with zipfile.ZipFile(path, "w") as zf: + zf.writestr("boot_b.ic94", b"B") + zf.writestr("eeprom.ic1", b"E") + crc = {n: f"{zlib.crc32(d):08x}" for n, d in (("a", b"A"), ("b", b"B"), ("e", b"E"))} + members = [ + {"crc32": crc["a"], "required": True, "group": "boot"}, + {"crc32": crc["b"], "required": True, "group": "boot"}, + {"crc32": crc["e"], "required": True}, + ] + self.assertTrue(holds_required_members(str(path), members)) + self.assertFalse(holds_required_members(str(path), members[:1] + members[2:])) + + def test_schema_refuses_a_set_split_on_required(self): + from validate_schemas import _group_requirement_split, ROOT + + path = ROOT / "emulators" / "seq.yml" + split = {"files": [{"name": "a", "group": "g", "required": True}, + {"name": "b", "group": "g", "required": False}]} + self.assertEqual(len(_group_requirement_split(path, split)), 1) + self.assertEqual(_group_requirement_split(path, {"files": self.profile["files"]}), []) + if __name__ == "__main__": unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index feb7414ce..0fa2b6dc3 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -603,7 +603,7 @@ look. | Field | Description | |-------|-------------| | `name` | filename as the core expects it | -| `required` | true if the core needs this file to function | +| `required` | true if the core needs this file to function. On a member of a `group`, the requirement of the set | | `system` | system ID this file belongs to (for multi-system profiles) | | `systems` | the systems of the profile this file serves, when it serves several but not all (a MAME device set the machines of two systems load). Never written with `system` | | `archive` | ROM set the file lives inside, for arcade entries (e.g. `neogeo.zip`) | @@ -621,7 +621,8 @@ look. | `region` | always a list of territory slugs from the schema enum (`[north-america]`, `[japan, asia-ntsc]`). The region the code selects the file **for**, never the region of the dump. See step 3b | | `region_check` | true if the core refuses to boot a game whose region does not match the BIOS | | `fast_boot` | BIOS generation label the core uses to decide whether fast boot is available | -| `priority` | preference order when several BIOS satisfy the same slot, **lowest wins**. DuckStation's `FindBIOSImageInDirectory` keeps the image whose priority is lower, and its table de-prioritizes by raising the number: the buggy launch console sits at 50 and PS2 images at 100, while `scph5501` sits at 5. A core that walks an ordered search list, as PicoDrive does with `biosfiles_us/eu/jp`, maps onto the same field as 1, 2, 3. Only set it from the source. `--one-per-slot` reads it | +| `priority` | preference order when several BIOS satisfy the same slot, **lowest wins**. DuckStation's `FindBIOSImageInDirectory` keeps the image whose priority is lower, and its table de-prioritizes by raising the number: the buggy launch console sits at 50 and PS2 images at 100, while `scph5501` sits at 5. A core that walks an ordered search list, as PicoDrive does with `biosfiles_us/eu/jp`, maps onto the same field as 1, 2, 3. Only set it from the source. `--one-per-slot` reads it, comparing the members of one `group` | +| `group` | entries the code takes one of: it loads the first it finds (Yabause tries `saturn_bios.bin`, `sega_101.bin`, `mpr-17933.bin`), and holding any member fills the need. Verify, the gap report and the packs report a set nobody holds as one gap under its first member, and no member of a held set as missing. `required` is the same on every member, the order stays in `priority`. A list puts a file in several sets. Where the code keeps a list per region, each list is its own group (PicoDrive's `mega-cd-us`, `mega-cd-eu`, `mega-cd-jp`) | | `embedded` | true if the data is compiled into the binary and the external file is optional | | `bundled` | true if the emulator's own release installs the file where it reads it, for the build `mode` names (all builds when `mode` is absent). Packs and the installer leave it out: the user already has it. A file the standalone release ships but the libretro core does not (PPSSPP's assets, Dolphin's Sys) is bundled with `mode: standalone` only | | `not_bundled_on` | platforms (registry keys) whose package leaves out a file the release bundles: their packs bring it (Batocera installs hcl without its data) |