mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-12 21:13:49 -06:00
feat: read a search list as one need
A core that loads the first of several names (yabause: saturn_bios.bin, sega_101.bin, mpr-17933.bin) needs one of them. Profiles wrote this as required on the first name and optional on the others, so verify and the gap report called saturn_bios.bin a missing required file while sega_101.bin was held, and a set held only through a name marked optional read as a need met by an optional file. Model: an explicit group on the file entry, mirroring the platform field. priority was not reused: it is a rank, read by slot.py, region.py and truth.py inside a (system, region) slot, and that slot is not the set. px68k-sdl ranks its IPL list and its font list each from 1 in one system, pom2 ranks one list per machine model, DuckStation ranks 105 images across every region and takes any of them. A rank cannot say where one list ends. priority stays the order inside a group. Readers, through alternatives.settle_sets: - verify --emulator and the emulator pack: a held member fills the set, the others leave the report, a set nobody holds is one gap under its first member naming the others. - find_undeclared_files (verify platform, core complement of packs and manifests): same rule, a platform declaration or launcher counts as answering the set. - cross_reference: excused members are set aside as other versions. - slot.py: ranks compare members of one set only. - archive members: a required set inside an archive is met by one. - truth: a member gives no requirement of its own, one_of and one_of_required carry the set, and the exporters and the diff leave such a line's required as the platform wrote it. - validate_schemas: members of a group share required.
This commit is contained in:
1 parent
a40f8416e1
commit
2c76cc7df9
13 files changed
+599
-41
No files matched your search
@@ -0,0 +1,87 @@
|
||||
"""Entries a core accepts in place of one another.
|
||||
|
||||
A profile writes ``group:`` on the entries of one search list. Yabause tries
|
||||
saturn_bios.bin, sega_101.bin, then mpr-17933.bin and loads the first it
|
||||
finds (libretro.c:1035-1049): it needs one of them and none in particular.
|
||||
``required`` on a member states the set's requirement, and ``priority`` keeps
|
||||
the order the code reads them in.
|
||||
|
||||
Holding one member fills the set, so the other members are not missing. A
|
||||
set nobody holds is one gap, reported under its first member, never one gap
|
||||
per member. Every reader settles sets through ``settle_sets`` so that verify,
|
||||
the gap report and the packs agree on what is missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Hashable, Iterable, Sequence
|
||||
|
||||
_UNRANKED = 1 << 30
|
||||
|
||||
|
||||
def groups_of(entry: dict) -> tuple[str, ...]:
|
||||
"""The sets an entry belongs to: one name, or a list for a file two
|
||||
machine models both fall back on."""
|
||||
value = entry.get("group")
|
||||
if not value:
|
||||
return ()
|
||||
values = value if isinstance(value, list) else [value]
|
||||
return tuple(str(v) for v in values if v)
|
||||
|
||||
|
||||
def set_keys(owner: Hashable, entry: dict) -> tuple[tuple, ...]:
|
||||
"""The keys of an entry's sets, scoped to its owner (a profile, or a
|
||||
profile under one build)."""
|
||||
return tuple((owner, group) for group in groups_of(entry))
|
||||
|
||||
|
||||
def slot_set_keys(owner: Hashable, entries: Iterable[dict]) -> tuple[tuple, ...]:
|
||||
"""The sets the entries on one destination belong to. A destination has
|
||||
one outcome however many versions it lists, so its versions' sets are
|
||||
settled together."""
|
||||
keys: list[tuple] = []
|
||||
for entry in entries:
|
||||
keys.extend(k for k in set_keys(owner, entry) if k not in keys)
|
||||
return tuple(keys)
|
||||
|
||||
|
||||
def _rank(entry: dict, position: int) -> tuple[int, int]:
|
||||
priority = entry.get("priority")
|
||||
return (priority if isinstance(priority, int) else _UNRANKED, position)
|
||||
|
||||
|
||||
def settle_sets(
|
||||
members: Sequence[tuple[tuple[tuple, ...], dict, bool]],
|
||||
) -> tuple[set[int], dict[int, list[str]]]:
|
||||
"""Which unheld members are excused, and which one stands for its set.
|
||||
|
||||
``members`` lists ``(set keys, entry, fills)`` in the order the profile
|
||||
declares them, where ``fills`` says the member is held (or answered by a
|
||||
declaration that reports it). Returns the positions to leave out of a
|
||||
report and, for each set no member fills, the position kept as its one
|
||||
gap with the names of the others. A member fills or not on its own: a
|
||||
held one is never excused, and an unheld one stays reported when it
|
||||
stands for at least one unfilled set.
|
||||
"""
|
||||
by_set: dict[tuple, list[int]] = {}
|
||||
for position, (keys, _entry, _fills) in enumerate(members):
|
||||
for key in keys:
|
||||
by_set.setdefault(key, []).append(position)
|
||||
|
||||
standing: dict[int, list[str]] = {}
|
||||
for positions in by_set.values():
|
||||
if any(members[p][2] for p in positions):
|
||||
continue
|
||||
first = min(positions, key=lambda p: _rank(members[p][1], p))
|
||||
names = standing.setdefault(first, [])
|
||||
for p in positions:
|
||||
name = str(members[p][1].get("name", ""))
|
||||
if p != first and name not in names:
|
||||
names.append(name)
|
||||
|
||||
excused = {
|
||||
position
|
||||
for position, (keys, _entry, fills) in enumerate(members)
|
||||
if keys and not fills and position not in standing
|
||||
}
|
||||
return excused, standing
|
||||
+36
-4
@@ -1267,7 +1267,39 @@ _archive_index: dict[int, tuple[list[dict], int, dict[str | None, list[dict]]]]
|
||||
|
||||
|
||||
def _member(f: dict) -> dict:
|
||||
return {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
|
||||
member = {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
|
||||
if f.get("group"):
|
||||
# One of the members sharing it is enough (alternatives.py).
|
||||
member["group"] = f["group"]
|
||||
return member
|
||||
|
||||
|
||||
def _required_met(members: list[dict], held: Callable[[dict], bool]) -> tuple[int, bool]:
|
||||
"""How many requirements an archive meets, and whether it meets all.
|
||||
|
||||
A required member is one requirement, and so is a required set of
|
||||
alternatives, met when one of its members is held: Odin reads the
|
||||
first of three Hikaru boot ROMs it finds in hikaru.zip.
|
||||
"""
|
||||
from alternatives import groups_of
|
||||
|
||||
met = 0
|
||||
complete = True
|
||||
sets: dict[str, bool] = {}
|
||||
for m in members:
|
||||
if not m["required"]:
|
||||
continue
|
||||
names = groups_of(m)
|
||||
if names:
|
||||
for name in names:
|
||||
sets[name] = sets.get(name, False) or held(m)
|
||||
continue
|
||||
if held(m):
|
||||
met += 1
|
||||
else:
|
||||
complete = False
|
||||
met += sum(sets.values())
|
||||
return met, complete and all(sets.values())
|
||||
|
||||
|
||||
def archive_members(archive: str, files: list[dict]) -> list[dict]:
|
||||
@@ -1316,7 +1348,7 @@ def holds_required_members(path: str, members: list[dict]) -> bool:
|
||||
"""Whether an archive holds every member a profile requires inside it."""
|
||||
crcs = zip_member_crcs(path)
|
||||
names = zip_member_names(path)
|
||||
return all(_member_held(m, crcs, names) for m in members if m["required"])
|
||||
return _required_met(members, lambda m: _member_held(m, crcs, names))[1]
|
||||
|
||||
|
||||
def contents_members(file_entry: dict) -> list[dict]:
|
||||
@@ -1360,8 +1392,8 @@ def _holding_members(paths: list[str], members: list[dict]) -> list[str]:
|
||||
def held(path: str) -> tuple[int, int]:
|
||||
crcs = zip_member_crcs(path)
|
||||
names = zip_member_names(path)
|
||||
found = [m for m in members if _member_held(m, crcs, names)]
|
||||
return sum(1 for m in found if m["required"]), len(found)
|
||||
found = {id(m) for m in members if _member_held(m, crcs, names)}
|
||||
return _required_met(members, lambda m: id(m) in found)[0], len(found)
|
||||
|
||||
scores = {path: held(path) for path in paths}
|
||||
best = max(scores.values(), default=(0, 0))
|
||||
|
||||
@@ -23,6 +23,7 @@ from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from alternatives import set_keys, settle_sets
|
||||
from common import (
|
||||
platform_arg,
|
||||
ProfileSelectionError,
|
||||
@@ -797,6 +798,9 @@ def _cross_reference_profile(
|
||||
# the profile's entries in their order, so a position names one entry.
|
||||
position = {id(f): i for view, _ in layouts for i, f in enumerate(view.get("files", []))}
|
||||
fates: dict[int, str] = {}
|
||||
# Grouped entries met, as (set keys, entry, fills, the list holding
|
||||
# its row, the row): settled once the profile is read.
|
||||
set_records: list[tuple[tuple, dict, bool, list | None, dict | None]] = []
|
||||
|
||||
def fate(f: dict, what: str) -> None:
|
||||
key = position[id(f)]
|
||||
@@ -834,15 +838,18 @@ def _cross_reference_profile(
|
||||
|
||||
# Collect unsourceable files separately (documented, not a gap)
|
||||
unsourceable_reason = f.get("unsourceable", "")
|
||||
keys = set_keys((emu_name, is_standalone), f)
|
||||
if unsourceable_reason:
|
||||
seen_files.add(seen_key)
|
||||
fate(f, "unsourceable")
|
||||
unsourceable_list.append({
|
||||
row = {
|
||||
"name": fname,
|
||||
"required": f.get("required", False),
|
||||
"reason": unsourceable_reason,
|
||||
"source_ref": f.get("source_ref", ""),
|
||||
})
|
||||
}
|
||||
unsourceable_list.append(row)
|
||||
set_records.append((keys, f, False, unsourceable_list, row))
|
||||
continue
|
||||
|
||||
if outside_gap_scope(f, is_standalone):
|
||||
@@ -863,6 +870,7 @@ def _cross_reference_profile(
|
||||
if in_platform:
|
||||
seen_files.add(seen_key)
|
||||
fate(f, "declared")
|
||||
set_records.append((keys, f, True, None, None))
|
||||
covered.append({
|
||||
"name": fname,
|
||||
"path": effective_path,
|
||||
@@ -955,6 +963,21 @@ def _cross_reference_profile(
|
||||
(emu_name, is_standalone, f, probe, entry)
|
||||
)
|
||||
gaps.append(entry)
|
||||
set_records.append((keys, f, in_repo, gaps, entry))
|
||||
|
||||
# One need per set: a member another one fills is not a gap, and a set
|
||||
# nobody holds is one gap under its first member (alternatives.py), as
|
||||
# verify and the packs count it.
|
||||
grouped = [r for r in set_records if r[0]]
|
||||
excused, standing = settle_sets([(keys, f, fills) for keys, f, fills, _l, _r in grouped])
|
||||
for at, (_keys, member, _fills, rows, set_row) in enumerate(grouped):
|
||||
if set_row is None or rows is None:
|
||||
continue
|
||||
if at in standing:
|
||||
set_row["alternatives"] = standing[at]
|
||||
if at in excused:
|
||||
rows.remove(set_row)
|
||||
fates[position[id(member)]] = "other_versions"
|
||||
|
||||
for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]):
|
||||
gaps.append(ag)
|
||||
@@ -1333,6 +1356,8 @@ def print_report(
|
||||
fc = g.get("archive_file_count", 0)
|
||||
rc = g.get("archive_required_count", 0)
|
||||
archive_info = f" ({fc} files, {rc} required)"
|
||||
if g.get("alternatives") and not g.get("in_repo"):
|
||||
archive_info += f" (or {', '.join(g['alternatives'])})"
|
||||
print(f" {req} {g['name']} [{src}]{archive_info}{note}")
|
||||
|
||||
total_gaps += gaps
|
||||
|
||||
@@ -264,6 +264,8 @@ class NativeFile:
|
||||
return bool(self.truth["required"])
|
||||
if self.platform is not None and self.platform.get("required") is not None:
|
||||
return bool(self.platform["required"])
|
||||
if self.truth is not None and self.truth.get("one_of_required") is not None:
|
||||
return bool(self.truth["one_of_required"])
|
||||
return True
|
||||
|
||||
@property
|
||||
@@ -645,10 +647,15 @@ def build_native_model(
|
||||
matched.filled.append(field_name)
|
||||
t_req = truth_entry.get("required")
|
||||
p_req = (matched.platform or {}).get("required")
|
||||
# A file a core takes as one of a set has no requirement of
|
||||
# its own: a platform line cannot say "one of these", so
|
||||
# truth leaves its value as written rather than claim
|
||||
# every member optional or every member needed.
|
||||
if (
|
||||
t_req is not None
|
||||
and p_req is not None
|
||||
and bool(t_req) != bool(p_req)
|
||||
and not (truth_entry.get("one_of_required") and not t_req)
|
||||
):
|
||||
matched.corrections.append("required")
|
||||
report.required_corrected.append(
|
||||
|
||||
@@ -189,16 +189,21 @@ class Exporter(BaseExporter):
|
||||
} - {""}
|
||||
readers: set[str] = set()
|
||||
requirers: set[str] = set()
|
||||
set_readers: set[str] = set()
|
||||
for entry in entries:
|
||||
cores = set(entry.get("_cores") or ()) & named
|
||||
readers |= cores
|
||||
if entry.get("one_of_required"):
|
||||
set_readers |= cores & set(entry.get("one_of") or ())
|
||||
required_by = entry.get("_required_by")
|
||||
if required_by is None:
|
||||
if entry.get("required") is not False:
|
||||
requirers |= cores
|
||||
else:
|
||||
requirers |= cores & set(required_by)
|
||||
if not readers:
|
||||
# A core that needs one file of a set needs none of them on its own,
|
||||
# which a mandatory flag cannot say: the line keeps its value.
|
||||
if not readers or (not requirers and set_readers):
|
||||
if fe.platform is not None and fe.platform.get("required") is not None:
|
||||
return bool(fe.platform["required"])
|
||||
return fe.required
|
||||
|
||||
@@ -36,6 +36,7 @@ import zipfile
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from alternatives import settle_sets, slot_set_keys
|
||||
from common import resolve_platform_cores as _platform_cores
|
||||
from common import (
|
||||
installed_data_refs,
|
||||
@@ -1838,6 +1839,10 @@ def generate_emulator_pack(
|
||||
continue
|
||||
slots.setdefault(dest, []).append(fe)
|
||||
|
||||
# What each destination came to, settled by set once the loop
|
||||
# is over: a member another member fills is not missing, and a
|
||||
# set nobody holds is reported once (alternatives.py).
|
||||
outcomes: list[tuple[list[dict], bool, Callable[[], None] | None]] = []
|
||||
for dest, alternatives in slots.items():
|
||||
# Name order, first file wins, case folded: verify replays
|
||||
# the rule and reports a later profile's file as missing.
|
||||
@@ -1923,17 +1928,25 @@ def generate_emulator_pack(
|
||||
held, packed_at.get(dest.lower())
|
||||
):
|
||||
# The file already at this path, asked for twice.
|
||||
outcomes.append((alternatives, True, None))
|
||||
continue
|
||||
report_missing(alternatives[0]["name"], dest)
|
||||
outcomes.append((alternatives, False, functools.partial(
|
||||
report_missing, alternatives[0]["name"], dest
|
||||
)))
|
||||
continue
|
||||
|
||||
if chosen is None:
|
||||
if refusal:
|
||||
rejected_files.append(refusal)
|
||||
outcomes.append((alternatives, False, None))
|
||||
elif all(fe.get("unsourceable") for fe in alternatives):
|
||||
unsourceable_files.append(alternatives[0]["name"])
|
||||
outcomes.append((alternatives, False, functools.partial(
|
||||
unsourceable_files.append, alternatives[0]["name"]
|
||||
)))
|
||||
else:
|
||||
report_missing(alternatives[0]["name"], dest)
|
||||
outcomes.append((alternatives, False, functools.partial(
|
||||
report_missing, alternatives[0]["name"], dest
|
||||
)))
|
||||
continue
|
||||
|
||||
fe, local_path, how = chosen
|
||||
@@ -1954,8 +1967,11 @@ def generate_emulator_pack(
|
||||
total_files += 1
|
||||
if record is not None:
|
||||
record.files.append(dest)
|
||||
outcomes.append((alternatives, True, None))
|
||||
else:
|
||||
missing_files.append(fe["name"])
|
||||
outcomes.append((alternatives, False, functools.partial(
|
||||
missing_files.append, fe["name"]
|
||||
)))
|
||||
finally:
|
||||
if os.path.exists(tmp_path):
|
||||
os.unlink(tmp_path)
|
||||
@@ -1969,6 +1985,7 @@ def generate_emulator_pack(
|
||||
# (but allow same file to be packed under different destinations,
|
||||
# e.g., IPL.bin in GC/USA/ and GC/EUR/ from same source)
|
||||
dedup_key_hash = f"{os.path.realpath(local_path)}:{dest}"
|
||||
outcomes.append((alternatives, True, None))
|
||||
if dedup_key_hash in seen_hashes:
|
||||
continue
|
||||
seen_hashes.add(dedup_key_hash)
|
||||
@@ -1985,6 +2002,14 @@ def generate_emulator_pack(
|
||||
if record is not None:
|
||||
record.files.append(dest)
|
||||
|
||||
excused, _standing = settle_sets([
|
||||
(slot_set_keys(emu_name, entries), entries[0], fills)
|
||||
for entries, fills, _report in outcomes
|
||||
])
|
||||
for position, (_entries, _fills, report) in enumerate(outcomes):
|
||||
if report is not None and position not in excused:
|
||||
report()
|
||||
|
||||
# The trees after the files, as in a platform pack: a dump the
|
||||
# profile declares wins over the copy a tree bundles at the same
|
||||
# path (Dolphin's real dsp_rom.bin over Sys/GC's free one).
|
||||
|
||||
+11
-4
@@ -29,6 +29,8 @@ rank: the second core reads it by its own rule, which no rank expresses.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from alternatives import set_keys
|
||||
|
||||
BIOS_CATEGORY = "bios"
|
||||
|
||||
|
||||
@@ -65,9 +67,11 @@ def build_slot_index(profiles: dict) -> dict[str, dict]:
|
||||
entry = index.setdefault(
|
||||
key,
|
||||
{"rank": None, "conflict": False, "regions": set(),
|
||||
"emulators": [], "ranked_by": set(), "unranked_by": set()},
|
||||
"emulators": [], "ranked_by": set(), "unranked_by": set(),
|
||||
"sets": set()},
|
||||
)
|
||||
entry["regions"] |= {str(r) for r in (f.get("region") or [])}
|
||||
entry["sets"] |= set(set_keys(emu_name, f))
|
||||
rank = f.get("priority")
|
||||
entry["unranked_by" if rank is None else "ranked_by"].add(emu_name)
|
||||
if rank is not None:
|
||||
@@ -123,7 +127,7 @@ def resolve_slot_drops(
|
||||
# A slot is a system AND a declared region: the Japanese and American
|
||||
# PlayStation BIOS are not alternatives to each other, and comparing their
|
||||
# ranks across regions would pick one territory's file for another's.
|
||||
tiers: dict[tuple[str, tuple[str, ...]], list[tuple[str, int | None]]] = {}
|
||||
tiers: dict[tuple[str, tuple[str, ...], tuple], list[tuple[str, int | None]]] = {}
|
||||
for group_id, members in groups.items():
|
||||
for destination, name in members:
|
||||
if held is not None and destination not in held:
|
||||
@@ -132,11 +136,14 @@ def resolve_slot_drops(
|
||||
if entry is None:
|
||||
keep.add(destination)
|
||||
continue
|
||||
# A rank orders the members of one set: px68k-sdl ranks its IPL
|
||||
# and its font ROM lists each from 1, and the two never compete.
|
||||
tier = tuple(sorted(entry["regions"]))
|
||||
sets = tuple(sorted(entry.get("sets") or ()))
|
||||
rank = None if entry.get("conflict") else entry["rank"]
|
||||
tiers.setdefault((group_id, tier), []).append((destination, rank))
|
||||
tiers.setdefault((group_id, tier, sets), []).append((destination, rank))
|
||||
|
||||
for (group_id, tier), candidates in tiers.items():
|
||||
for (group_id, tier, _sets), candidates in tiers.items():
|
||||
if len(candidates) < 2:
|
||||
keep.update(dest for dest, _p in candidates)
|
||||
continue
|
||||
|
||||
+32
-3
@@ -18,6 +18,7 @@ from common import (
|
||||
resolve_platform_cores,
|
||||
standalone_set as platform_standalone_set,
|
||||
)
|
||||
from alternatives import groups_of
|
||||
from hashing import compute_hashes
|
||||
from validation import (
|
||||
filter_files_by_mode,
|
||||
@@ -276,11 +277,15 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None:
|
||||
existing["_source_refs"] = existing.get("_source_refs", set()) | {sr_key}
|
||||
else:
|
||||
existing.setdefault("_source_refs", set())
|
||||
if file_entry.get("required"):
|
||||
if groups_of(file_entry):
|
||||
_fold_set_member(existing, file_entry, emu_name)
|
||||
elif file_entry.get("required"):
|
||||
existing["required"] = True
|
||||
# Which cores need it: `required` above is the union, true when
|
||||
# any core needs it, and a package of one core must not read it.
|
||||
existing["_required_by"] = existing.get("_required_by", set()) | {emu_name}
|
||||
elif file_entry.get("required") is not None:
|
||||
existing.setdefault("required", file_entry["required"])
|
||||
_merge_hashes(existing, file_entry, emu_name, revision)
|
||||
# Merge non-hash data fields if existing lacks them.
|
||||
# A core that creates an entry without size/path/validation may be
|
||||
@@ -292,6 +297,21 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None:
|
||||
_merge_region(existing, file_entry.get("region"))
|
||||
|
||||
|
||||
def _fold_set_member(entry: dict, file_entry: dict, emu_name: str) -> None:
|
||||
"""A core that takes this file as one of a set needs the set, not it.
|
||||
|
||||
Yabause loads the first of saturn_bios.bin, sega_101.bin and
|
||||
mpr-17933.bin it finds. None of the three is required on its own, so
|
||||
the core stays out of `required` and `_required_by`, and `_one_of`
|
||||
names it with `one_of_required` saying whether its set is required.
|
||||
"""
|
||||
entry["_one_of"] = entry.get("_one_of", set()) | {emu_name}
|
||||
if file_entry.get("required", True):
|
||||
entry["one_of_required"] = True
|
||||
else:
|
||||
entry.setdefault("one_of_required", False)
|
||||
|
||||
|
||||
def _merge_region(existing: dict, theirs: str | list | None) -> None:
|
||||
"""The regions every reader selects the file for, the pack's own rule.
|
||||
|
||||
@@ -379,7 +399,8 @@ def _merge_priority(existing: dict, theirs: int | None) -> None:
|
||||
|
||||
def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
|
||||
entry: dict = {"name": file_entry["name"]}
|
||||
if file_entry.get("required") is not None:
|
||||
grouped = bool(groups_of(file_entry))
|
||||
if file_entry.get("required") is not None and not grouped:
|
||||
entry["required"] = file_entry["required"]
|
||||
for field in _COPIED_FIELDS:
|
||||
val = file_entry.get(field)
|
||||
@@ -396,7 +417,9 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
|
||||
if not file_entry.get("region"):
|
||||
entry["_region_void"] = True
|
||||
entry["_cores"] = {emu_name}
|
||||
entry["_required_by"] = {emu_name} if file_entry.get("required") else set()
|
||||
entry["_required_by"] = {emu_name} if file_entry.get("required") and not grouped else set()
|
||||
if grouped:
|
||||
_fold_set_member(entry, file_entry, emu_name)
|
||||
entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set()
|
||||
entry["_brought_by"] = _brought(file_entry, emu_name)
|
||||
sr = file_entry.get("source_ref")
|
||||
@@ -848,6 +871,9 @@ def generate_platform_truth(
|
||||
fe.pop("_region_void", None)
|
||||
fe["_cores"] = sorted(fe.get("_cores", set()))
|
||||
fe["_required_by"] = sorted(fe.get("_required_by", set()))
|
||||
one_of = fe.pop("_one_of", None)
|
||||
if one_of:
|
||||
fe["one_of"] = sorted(one_of)
|
||||
fe["_named_by"] = sorted(fe.get("_named_by", set()))
|
||||
fe["_brought_by"] = sorted(fe.get("_brought_by", set()))
|
||||
if not fe["_brought_by"]:
|
||||
@@ -950,6 +976,9 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
hash_mismatch.append(mismatch)
|
||||
t_req = t_entry.get("required")
|
||||
s_req = s_entry.get("required")
|
||||
if t_entry.get("one_of_required") and not t_req:
|
||||
# Needed as one of a set: no value of its own to compare.
|
||||
t_req = None
|
||||
if t_req is not None and s_req is not None and t_req != s_req:
|
||||
required_mismatch.append(
|
||||
{
|
||||
|
||||
@@ -86,6 +86,7 @@ def _validate_yaml_directory(
|
||||
out.extend(_save_dir_without_library_name(path, data))
|
||||
out.extend(_systems_outside_profile(path, data))
|
||||
out.extend(_doubled_paths(path))
|
||||
out.extend(_group_requirement_split(path, data))
|
||||
return out
|
||||
|
||||
|
||||
@@ -122,6 +123,30 @@ def _systems_outside_profile(path: Path, document: object) -> list[str]:
|
||||
return out
|
||||
|
||||
|
||||
def _group_requirement_split(path: Path, document: object) -> list[str]:
|
||||
"""A group whose members disagree on required.
|
||||
|
||||
The members of a group are one need, so required states the set's and
|
||||
reads the same on each. A true on the first name only wrote the search
|
||||
order into required, and the other names read as optional files.
|
||||
"""
|
||||
from alternatives import groups_of
|
||||
|
||||
if not isinstance(document, dict):
|
||||
return []
|
||||
seen: dict[str, set[bool]] = {}
|
||||
for entry in document.get("files") or []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
for group in groups_of(entry):
|
||||
seen.setdefault(group, set()).add(bool(entry.get("required", True)))
|
||||
return [
|
||||
f"{path.relative_to(ROOT)}: group {group} mixes required true and false"
|
||||
for group, values in sorted(seen.items())
|
||||
if len(values) > 1
|
||||
]
|
||||
|
||||
|
||||
def _checks_without_values(path: Path, document: object) -> list[str]:
|
||||
"""A check an entry declares with nothing to compare against.
|
||||
|
||||
|
||||
+114
-20
@@ -37,6 +37,7 @@ import slots
|
||||
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
from hashing import parse_md5_list
|
||||
from alternatives import set_keys, settle_sets, slot_set_keys
|
||||
from common import (
|
||||
_foreign_tree,
|
||||
_owner_tokens,
|
||||
@@ -534,6 +535,9 @@ def find_undeclared_files(
|
||||
config, profiles, relevant, db, declared_destinations(config)
|
||||
)
|
||||
launcher_candidates: list[tuple[str, bool, dict, dict]] = []
|
||||
# Each grouped requirement met, as (set keys, entry, fills, (reported
|
||||
# entry, the entry it is a claim on)): settled once every core is read.
|
||||
set_records: list[tuple[tuple, dict, bool, tuple[dict, dict | None] | None]] = []
|
||||
undeclared = []
|
||||
seen_files: set[tuple] = set()
|
||||
# Key -> the entry reported for it. A later profile reading the same
|
||||
@@ -606,6 +610,9 @@ def find_undeclared_files(
|
||||
)
|
||||
if verdict == "settled":
|
||||
seen_files.add(seen_key)
|
||||
# Answered by the platform's own declaration, which its
|
||||
# report covers: the set it belongs to is spoken for.
|
||||
set_records.append((set_keys((emu_name, is_standalone), f), f, True, None))
|
||||
continue
|
||||
if verdict == "skip":
|
||||
continue
|
||||
@@ -761,12 +768,16 @@ def find_undeclared_files(
|
||||
entry["members"] = members
|
||||
if launcher_layer and not save_root:
|
||||
launcher_candidates.append((emu_name, is_standalone, f, entry))
|
||||
keys = set_keys((emu_name, is_standalone), f)
|
||||
if duplicate:
|
||||
first.setdefault("other_claims", []).append(entry)
|
||||
set_records.append((keys, f, in_repo, (entry, first)))
|
||||
continue
|
||||
reported[seen_key] = entry
|
||||
undeclared.append(entry)
|
||||
set_records.append((keys, f, in_repo, (entry, None)))
|
||||
|
||||
settled: set[int] = set()
|
||||
if launcher_layer:
|
||||
added, settled = launcher_layer.settle(
|
||||
launcher_candidates, lambda report: held_record(report, db)
|
||||
@@ -774,12 +785,63 @@ def find_undeclared_files(
|
||||
undeclared = [u for u in undeclared if id(u) not in settled]
|
||||
undeclared.extend(added)
|
||||
|
||||
undeclared = _settle_core_sets(undeclared, set_records, settled)
|
||||
|
||||
for entry in sorted(archive_entries.values(), key=lambda e: e["name"]):
|
||||
undeclared.append(entry)
|
||||
|
||||
return undeclared
|
||||
|
||||
|
||||
def _settle_core_sets(
|
||||
undeclared: list[dict],
|
||||
records: list[tuple[tuple, dict, bool, tuple[dict, dict | None] | None]],
|
||||
launcher_settled: set[int],
|
||||
) -> list[dict]:
|
||||
"""The core complement with each set of alternatives settled.
|
||||
|
||||
A member the collection lacks is dropped when another member is held,
|
||||
declared by the platform or placed by its launcher, and a set nobody
|
||||
holds keeps one entry, its first member, naming the others. A claim
|
||||
another profile shares stays: its own core may need that very file.
|
||||
"""
|
||||
grouped = [r for r in records if r[0]]
|
||||
if not grouped:
|
||||
return undeclared
|
||||
excused, standing = settle_sets([
|
||||
(keys, f, fills or (ref is not None and id(ref[0]) in launcher_settled))
|
||||
for keys, f, fills, ref in grouped
|
||||
])
|
||||
dropped: set[int] = set()
|
||||
for position, (_keys, _f, _fills, ref) in enumerate(grouped):
|
||||
if ref is None:
|
||||
continue
|
||||
entry, host = ref
|
||||
if position in standing:
|
||||
entry["alternatives"] = standing[position]
|
||||
if position not in excused:
|
||||
continue
|
||||
if host is not None:
|
||||
host["other_claims"] = [c for c in host.get("other_claims", []) if c is not entry]
|
||||
if not host["other_claims"]:
|
||||
del host["other_claims"]
|
||||
else:
|
||||
dropped.add(id(entry))
|
||||
out: list[dict] = []
|
||||
for entry in undeclared:
|
||||
if id(entry) not in dropped:
|
||||
out.append(entry)
|
||||
continue
|
||||
# Another core reading the same requirement still needs it.
|
||||
claims = entry.get("other_claims") or []
|
||||
if claims:
|
||||
promoted = dict(claims[0])
|
||||
if claims[1:]:
|
||||
promoted["other_claims"] = claims[1:]
|
||||
out.append(promoted)
|
||||
return out
|
||||
|
||||
|
||||
def find_exclusion_notes(
|
||||
config: dict,
|
||||
emulators_dir: str,
|
||||
@@ -1802,6 +1864,8 @@ def _print_undeclared_entry(
|
||||
name_label = f"{name} ({arc_count} file{'s' if arc_count != 1 else ''})"
|
||||
else:
|
||||
name_label = name
|
||||
if u.get("alternatives") and not u.get("in_repo"):
|
||||
name_label += f" (or {', '.join(u['alternatives'])})"
|
||||
print(f" {prefix}: {u['emulator']} needs {name_label}")
|
||||
checks = u.get("checks", [])
|
||||
if checks:
|
||||
@@ -2171,6 +2235,9 @@ def verify_emulator(
|
||||
return local_path, status
|
||||
|
||||
details = []
|
||||
# Members of a set another member fills, or that the set's first member
|
||||
# stands for: not missing, and out of every count.
|
||||
set_members: list[dict] = []
|
||||
file_status: dict[str, str] = {}
|
||||
file_severity: dict[str, str] = {}
|
||||
dest_to_name: dict[str, str] = {}
|
||||
@@ -2397,7 +2464,9 @@ def verify_emulator(
|
||||
# profile on one destination are the releases the code accepts
|
||||
# there (a shareware and a retail HOG, each GRP in a size table):
|
||||
# holding one of them fills the slot, and the others are not gaps.
|
||||
emu_results: dict[str, list[tuple[dict, bool, bool]]] = {}
|
||||
emu_results: dict[str, list[tuple[dict, bool, bool, dict]]] = {}
|
||||
placed_results: list[tuple[str, dict, bool, bool, list[dict]]] = []
|
||||
fallback_reason = "declared hash contradicted by the local file"
|
||||
|
||||
for file_entry in files:
|
||||
# Skip archive members (verified as archive units above)
|
||||
@@ -2497,13 +2566,13 @@ def verify_emulator(
|
||||
# The slot the emulator pack places this entry in: standalone_path
|
||||
# under --standalone, the sanitised path.
|
||||
dest = _resolve_destination(file_entry, standalone) or name
|
||||
emu_results.setdefault(dest, []).append((result, required, hle))
|
||||
emu_results.setdefault(dest, []).append((result, required, hle, file_entry))
|
||||
|
||||
for dest, alternatives in emu_results.items():
|
||||
# What the emulator pack places here: an accepted file, else a
|
||||
# file whose hash the profile contradicts. A refused copy is held
|
||||
# but never packed.
|
||||
fallback_reason = "declared hash contradicted by the local file"
|
||||
slot_entries = [a[3] for a in alternatives]
|
||||
if len(alternatives) > 1:
|
||||
# Reduced in the order generate_emulator_pack chooses, so the
|
||||
# destination's status is the one the pack reports: a slot it
|
||||
@@ -2556,27 +2625,49 @@ def verify_emulator(
|
||||
elif not block and packed is not None:
|
||||
occupy(dest, packed, emu_name)
|
||||
packed_dests.add(dest)
|
||||
for result, required, hle in alternatives:
|
||||
for result, required, hle, file_entry in alternatives:
|
||||
if block_reason and result["status"] != Status.MISSING:
|
||||
result["status"] = Status.MISSING
|
||||
result["reason"] = block_reason
|
||||
result["destination"] = dest
|
||||
details.append(result)
|
||||
placed_results.append((dest, result, required, hle, slot_entries))
|
||||
|
||||
# Aggregate by destination (path if available, else name)
|
||||
dest_to_name[dest] = result["name"]
|
||||
cur = result["status"]
|
||||
prev = file_status.get(dest)
|
||||
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(
|
||||
prev, 0
|
||||
):
|
||||
file_status[dest] = cur
|
||||
sev = _emulator_severity(cur, required, hle)
|
||||
prev_sev = file_severity.get(dest)
|
||||
if prev_sev is None or _SEVERITY_ORDER.get(
|
||||
sev, 0
|
||||
) > _SEVERITY_ORDER.get(prev_sev, 0):
|
||||
file_severity[dest] = sev
|
||||
# A set the code takes one member of is one need: once a member is
|
||||
# in the pack the others are not missing, and a set nobody holds is
|
||||
# one gap under its first member. Settled per profile, before the
|
||||
# destinations of several profiles are merged.
|
||||
excused, standing = settle_sets([
|
||||
(
|
||||
slot_set_keys(emu_name, entries),
|
||||
entries[0],
|
||||
result["status"] == Status.OK
|
||||
or (result["status"] == Status.UNTESTED
|
||||
and result.get("reason") == fallback_reason),
|
||||
)
|
||||
for _dest, result, _required, _hle, entries in placed_results
|
||||
])
|
||||
for position, (dest, result, required, hle, _fe) in enumerate(placed_results):
|
||||
if position in excused and result["status"] == Status.MISSING:
|
||||
set_members.append(result)
|
||||
continue
|
||||
if position in standing:
|
||||
result["alternatives"] = standing[position]
|
||||
details.append(result)
|
||||
|
||||
# Aggregate by destination (path if available, else name)
|
||||
dest_to_name[dest] = result["name"]
|
||||
cur = result["status"]
|
||||
prev = file_status.get(dest)
|
||||
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(
|
||||
prev, 0
|
||||
):
|
||||
file_status[dest] = cur
|
||||
sev = _emulator_severity(cur, required, hle)
|
||||
prev_sev = file_severity.get(dest)
|
||||
if prev_sev is None or _SEVERITY_ORDER.get(
|
||||
sev, 0
|
||||
) > _SEVERITY_ORDER.get(prev_sev, 0):
|
||||
file_severity[dest] = sev
|
||||
|
||||
occupy_trees(emu_name, profile)
|
||||
|
||||
@@ -2610,6 +2701,7 @@ def verify_emulator(
|
||||
"severity_counts": counts,
|
||||
"status_counts": status_counts,
|
||||
"details": details,
|
||||
"set_members": set_members,
|
||||
"data_dir_notices": sorted(set(data_dir_notices)),
|
||||
"ground_truth_coverage": {
|
||||
"with_validation": with_validation,
|
||||
@@ -2715,7 +2807,9 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
|
||||
seen.add(d["name"])
|
||||
req = "required" if d.get("required", True) else "optional"
|
||||
hle = ", HLE available" if d.get("hle_fallback") else ""
|
||||
print(f" MISSING ({req}{hle}): {d['name']}")
|
||||
others = d.get("alternatives")
|
||||
either = f" (or {', '.join(others)})" if others else ""
|
||||
print(f" MISSING ({req}{hle}): {d['name']}{either}")
|
||||
gt = d.get("ground_truth", [])
|
||||
if gt:
|
||||
if verbose:
|
||||
|
||||
Reference in new issue
Block a user