kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-11 20:43:50 -06:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge one-of sets

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-11-09 18:33:21 +01:00
2 parents 2fe15e99f9 50f9b642a0
commit ad424f796b
41 files changed
+1657 -449
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/alternatives.py
+212
View File
@@ -0,0 +1,212 @@
"""Entries a core accepts in place of one another.
A profile writes ``group:`` on the entries of one search list. Yabause tries
saturn_bios.bin, sega_101.bin, then mpr-17933.bin and loads the first it
finds (libretro.c:1035-1049): it needs one of them and none in particular.
The requirement belongs to the set, in the profile's ``groups:`` map, and a
member's ``required`` is derived from the sets it is in: true when one of
them is required. A fallback two machine models share (pom2's apple2.rom)
then needs no value of its own. ``priority`` keeps the order the code reads
the members in.
Holding one member fills the set, so the other members are not missing. A
set nobody holds is one gap, reported under its first member with the set's
requirement, never one gap per member. A --required-only build keeps one
member of a required set, the first the code would load among those held.
Every reader goes through this module so that verify, the gap report, the
packs and the manifests agree.
"""
from __future__ import annotations
from collections.abc import Callable, Hashable, Iterable, Sequence
from dataclasses import dataclass
_UNRANKED = 1 << 30
def groups_of(entry: dict) -> tuple[str, ...]:
"""The sets an entry belongs to: one name, a list for a file two machine
models both fall back on, or a map from set to the entry's rank there."""
value = entry.get("group")
if not value:
return ()
values = value if isinstance(value, (list, dict)) else [value]
return tuple(str(v) for v in values if v)
def rank_in(entry: dict, group: str) -> int | None:
"""The rank of an entry in one of its sets: the map's value when the
entry ranks differently in each (pom2's apple2.rom is second for the ][
and third at boot), else its priority."""
value = entry.get("group")
if isinstance(value, dict) and isinstance(value.get(group), int):
return value[group]
priority = entry.get("priority")
return priority if isinstance(priority, int) else None
def set_requirements(profile: dict) -> dict[str, bool]:
"""The requirement of each set the profile declares."""
declared = profile.get("groups") or {}
return {
str(name): bool((spec or {}).get("required", True))
for name, spec in declared.items()
}
def derive_member_requirements(profile: dict) -> None:
"""Write on each member the requirement its sets give it.
Read once when a profile is loaded, so every reader sees ``required``
as it always has. A member is required when one of its sets is.
"""
requirements = set_requirements(profile)
for entry in profile.get("files") or []:
if not isinstance(entry, dict):
continue
names = groups_of(entry)
if names and "required" not in entry:
entry["required"] = any(requirements.get(n, True) for n in names)
def required_groups(entry: dict, profile: dict) -> list[str]:
"""The required sets of its profile an entry is a member of."""
requirements = set_requirements(profile)
return [n for n in groups_of(entry) if requirements.get(n, True)]
def set_fields(entry: dict, profile: dict) -> dict:
"""What a --required-only build reads on a core file to keep one member
of a set: the entry's required sets, each with its rank there."""
sets = required_groups(entry, profile)
if not sets:
return {}
return {"required_groups": {group: rank_in(entry, group) for group in sets}}
def set_keys(owner: Hashable, entry: dict) -> tuple[tuple, ...]:
"""The keys of an entry's sets, scoped to its owner (a profile, or a
profile under one build)."""
return tuple((owner, group) for group in groups_of(entry))
def slot_set_keys(owner: Hashable, entries: Iterable[dict]) -> tuple[tuple, ...]:
"""The sets the entries on one destination belong to. A destination has
one outcome however many versions it lists, so its versions' sets are
settled together."""
keys: list[tuple] = []
for entry in entries:
keys.extend(k for k in set_keys(owner, entry) if k not in keys)
return tuple(keys)
def _rank(rank: int | None, position: int) -> tuple[int, int]:
return (rank if isinstance(rank, int) else _UNRANKED, position)
@dataclass
class Standing:
"""The member kept as the one gap of the sets nobody holds."""
others: list[str]
required: bool
def settle_sets(
members: Sequence[tuple[tuple[tuple, ...], dict, bool]],
set_required: Callable[[tuple], bool] = lambda _key: True,
) -> tuple[set[int], dict[int, Standing]]:
"""Which unheld members are excused, and which one stands for its set.
``members`` lists ``(set keys, entry, fills)`` in the order the profile
declares them, where ``fills`` says the member is held (or answered by a
declaration that reports it). Returns the positions to leave out of a
report and, for each set no member fills, the position kept as its one
gap with the names of the others and the set's requirement. A member
fills or not on its own: a held one is never excused, and an unheld one
stays reported when it stands for at least one unfilled set.
"""
by_set: dict[tuple, list[int]] = {}
for position, (keys, _entry, _fills) in enumerate(members):
for key in keys:
by_set.setdefault(key, []).append(position)
standing: dict[int, Standing] = {}
for key, positions in by_set.items():
if any(members[p][2] for p in positions):
continue
first = min(positions, key=lambda p: _rank(rank_in(members[p][1], key[1]), p))
held_up = standing.setdefault(first, Standing([], False))
held_up.required = held_up.required or set_required(key)
for p in positions:
name = str(members[p][1].get("name", ""))
if p != first and name not in held_up.others:
held_up.others.append(name)
excused = {
position
for position, (keys, _entry, fills) in enumerate(members)
if keys and not fills and position not in standing
}
return excused, standing
def required_only_spares(
entries: Sequence[dict],
owner: Callable[[dict], Hashable],
held: Callable[[dict], bool],
gone: Callable[[dict], bool] = lambda _entry: False,
sets: Callable[[dict], dict[str, int | None]] = lambda e: e.get("required_groups") or {},
) -> set[int]:
"""The ids of the held members a --required-only build leaves out.
``sets`` maps the required sets of an entry to its rank in each: a core
file of a platform pack carries them as ``required_groups``.
A required set needs one file: the build keeps the member the code
would load first among those it holds and still carries (``gone`` says
a region filter took it), by ``priority`` then declaration order, and
drops the others. A member of an optional set is not touched here,
nor one that another required set of its own still needs.
"""
candidates: dict[tuple, list[tuple[tuple[int, int], int]]] = {}
for position, entry in enumerate(entries):
if not held(entry) or gone(entry):
continue
for group, rank in sets(entry).items():
candidates.setdefault((owner(entry), group), []).append(
(_rank(rank, position), position)
)
kept: set[int] = set()
members: set[int] = set()
for ranked in candidates.values():
members.update(position for _rank_key, position in ranked)
kept.add(min(ranked)[1])
return {id(entries[p]) for p in members - kept}
def core_file_spares(
entries: Sequence[dict], gone: Callable[[dict], bool] = lambda _entry: False,
) -> set[int]:
"""The core files of a platform pack a --required-only build leaves out.
A set the platform declares a member of is carried by that declaration,
so its other members are spares. Otherwise one held member per required
set stays, as in an emulator pack. Read by the builder, the manifest
and verify alike.
"""
answered = {id(e) for e in entries if e.get("set_declared")}
# An archive under the core's own folder is read before the same name
# at the root (Kronos), so archive units rank first among equals.
rest = sorted(
(e for e in entries if id(e) not in answered),
key=lambda e: not e.get("archive"),
)
return answered | required_only_spares(
rest,
owner=lambda e: e.get("source_profile") or e.get("profile") or "",
held=lambda e: e.get("in_repo", True) is not False,
gone=gone,
)
scripts/common.py
+39 -4
View File
@@ -22,6 +22,8 @@ import zipfile
from pathlib import Path
from typing import Any, TextIO
from alternatives import derive_member_requirements
# Every build script imports this module. verify.py, slots.py, validate_pr.py
# and cross_reference.py started under 3.8 and stopped halfway on a
# str.removeprefix AttributeError, which named no version.
@@ -1267,7 +1269,39 @@ _archive_index: dict[int, tuple[list[dict], int, dict[str | None, list[dict]]]]
def _member(f: dict) -> dict:
return {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
member = {"crc32": str(f["crc32"]).lower(), "required": bool(f.get("required", True))}
if f.get("group"):
# One of the members sharing it is enough (alternatives.py).
member["group"] = f["group"]
return member
def _required_met(members: list[dict], held: Callable[[dict], bool]) -> tuple[int, bool]:
"""How many requirements an archive meets, and whether it meets all.
A required member is one requirement, and so is a required set of
alternatives, met when one of its members is held: Odin reads the
first of three Hikaru boot ROMs it finds in hikaru.zip.
"""
from alternatives import groups_of
met = 0
complete = True
sets: dict[str, bool] = {}
for m in members:
if not m["required"]:
continue
names = groups_of(m)
if names:
for name in names:
sets[name] = sets.get(name, False) or held(m)
continue
if held(m):
met += 1
else:
complete = False
met += sum(sets.values())
return met, complete and all(sets.values())
def archive_members(archive: str, files: list[dict]) -> list[dict]:
@@ -1316,7 +1350,7 @@ def holds_required_members(path: str, members: list[dict]) -> bool:
"""Whether an archive holds every member a profile requires inside it."""
crcs = zip_member_crcs(path)
names = zip_member_names(path)
return all(_member_held(m, crcs, names) for m in members if m["required"])
return _required_met(members, lambda m: _member_held(m, crcs, names))[1]
def contents_members(file_entry: dict) -> list[dict]:
@@ -1360,8 +1394,8 @@ def _holding_members(paths: list[str], members: list[dict]) -> list[str]:
def held(path: str) -> tuple[int, int]:
crcs = zip_member_crcs(path)
names = zip_member_names(path)
found = [m for m in members if _member_held(m, crcs, names)]
return sum(1 for m in found if m["required"]), len(found)
found = {id(m) for m in members if _member_held(m, crcs, names)}
return _required_met(members, lambda m: id(m) in found)[0], len(found)
scores = {path: held(path) for path in paths}
best = max(scores.values(), default=(0, 0))
@@ -2337,6 +2371,7 @@ def load_emulator_profiles(
if "emulator" not in profile:
continue
_normalize_hash_fields(profile)
derive_member_requirements(profile)
profiles[f.stem] = profile
_emulator_profiles_cache[cache_key] = profiles
return profiles
scripts/cross_reference.py
+50 -5
View File
@@ -23,6 +23,7 @@ from collections import Counter
from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__))
from alternatives import set_keys, set_requirements, settle_sets
from common import (
platform_arg,
ProfileSelectionError,
@@ -707,7 +708,7 @@ def slot_key(f: dict, standalone: bool = False) -> tuple:
f.get("archive"),
system_dir_path(f, where or fname),
f.get("system"),
f.get("variant_group"),
f.get("selection_slot"),
f.get("mode", "both"),
read_from_save_dir(f, standalone),
)
@@ -808,6 +809,10 @@ def _cross_reference_profile(
# the profile's entries in their order, so a position names one entry.
position = {id(f): i for view, _ in layouts for i, f in enumerate(view.get("files", []))}
fates: dict[int, str] = {}
# Grouped entries met, as (set keys, entry, fills, the list holding
# its row, the row): settled once the profile is read.
set_records: list[tuple[tuple, dict, bool, list | None, dict | None]] = []
archive_rows: list[dict] = []
def fate(f: dict, what: str) -> None:
key = position[id(f)]
@@ -845,15 +850,18 @@ def _cross_reference_profile(
# Collect unsourceable files separately (documented, not a gap)
unsourceable_reason = f.get("unsourceable", "")
keys = set_keys((emu_name, is_standalone), f)
if unsourceable_reason:
seen_files.add(seen_key)
fate(f, "unsourceable")
unsourceable_list.append({
row = {
"name": fname,
"required": f.get("required", False),
"reason": unsourceable_reason,
"source_ref": f.get("source_ref", ""),
})
}
unsourceable_list.append(row)
set_records.append((keys, f, False, unsourceable_list, row))
continue
if outside_gap_scope(f, is_standalone):
@@ -875,6 +883,7 @@ def _cross_reference_profile(
if in_platform:
seen_files.add(seen_key)
fate(f, "declared")
set_records.append((keys, f, True, None, None))
covered.append({
"name": fname,
"path": effective_path,
@@ -890,7 +899,7 @@ def _cross_reference_profile(
archive_key = (
archive,
f.get("system"),
f.get("variant_group"),
f.get("selection_slot"),
f.get("mode", "both"),
)
if archive_key not in archive_gaps:
@@ -912,6 +921,15 @@ def _cross_reference_profile(
"archive_required_count": 0,
}
entry = archive_gaps[archive_key]
if entry["archive_file_count"] == 0:
# The archive's own declaration names its sets: an archive
# is a member like a loose file (Kronos' stvbios.zip).
own = archive_entry(archive, emu_files)
set_records.append((
set_keys((emu_name, is_standalone), own), own,
entry["in_repo"], archive_rows, entry,
))
archive_rows.append(entry)
entry["archive_file_count"] += 1
if f.get("required", False):
entry["archive_required_count"] += 1
@@ -971,9 +989,34 @@ def _cross_reference_profile(
(emu_name, is_standalone, f, probe, entry)
)
gaps.append(entry)
set_records.append((keys, f, in_repo, gaps, entry))
# One need per set: a member another one fills is not a gap, and a set
# nobody holds is one gap under its first member (alternatives.py), as
# verify and the packs count it.
# Archive units rank first among equals, as the builder orders them.
grouped = sorted(
(r for r in set_records if r[0]), key=lambda r: r[3] is not archive_rows
)
requirements = set_requirements(profile)
excused, standing = settle_sets(
[(keys, f, fills) for keys, f, fills, _l, _r in grouped],
lambda key: requirements.get(key[1], True),
)
for at, (_keys, member, _fills, rows, set_row) in enumerate(grouped):
if set_row is None or rows is None:
continue
if at in standing:
set_row["alternatives"] = standing[at].others
set_row["required"] = standing[at].required
if at in excused:
rows.remove(set_row)
if id(member) in position:
fates[position[id(member)]] = "other_versions"
for ag in sorted(archive_gaps.values(), key=lambda e: e["name"]):
gaps.append(ag)
if any(ag is row for row in archive_rows):
gaps.append(ag)
tally = Counter(fates.values())
report[emu_name] = {
@@ -1432,6 +1475,8 @@ def print_report(
fc = g.get("archive_file_count", 0)
rc = g.get("archive_required_count", 0)
archive_info = f" ({fc} files, {rc} required)"
if g.get("alternatives") and not g.get("in_repo"):
archive_info += f" (or {', '.join(g['alternatives'])})"
print(f" {req} {g['name']} [{src}]{archive_info}{note}")
total_gaps += gaps
scripts/exporter/baseline.py
+7
View File
@@ -264,6 +264,8 @@ class NativeFile:
return bool(self.truth["required"])
if self.platform is not None and self.platform.get("required") is not None:
return bool(self.platform["required"])
if self.truth is not None and self.truth.get("one_of_required") is not None:
return bool(self.truth["one_of_required"])
return True
@property
@@ -645,10 +647,15 @@ def build_native_model(
matched.filled.append(field_name)
t_req = truth_entry.get("required")
p_req = (matched.platform or {}).get("required")
# A file a core takes as one of a set has no requirement of
# its own: a platform line cannot say "one of these", so
# truth leaves its value as written rather than claim
# every member optional or every member needed.
if (
t_req is not None
and p_req is not None
and bool(t_req) != bool(p_req)
and not (truth_entry.get("one_of_required") and not t_req)
):
matched.corrections.append("required")
report.required_corrected.append(
scripts/exporter/recalbox_exporter.py
+6 -1
View File
@@ -189,16 +189,21 @@ class Exporter(BaseExporter):
} - {""}
readers: set[str] = set()
requirers: set[str] = set()
set_readers: set[str] = set()
for entry in entries:
cores = set(entry.get("_cores") or ()) & named
readers |= cores
if entry.get("one_of_required"):
set_readers |= cores & set(entry.get("one_of") or ())
required_by = entry.get("_required_by")
if required_by is None:
if entry.get("required") is not False:
requirers |= cores
else:
requirers |= cores & set(required_by)
if not readers:
# A core that needs one file of a set needs none of them on its own,
# which a mandatory flag cannot say: the line keeps its value.
if not readers or (not requirers and set_readers):
if fe.platform is not None and fe.platform.get("required") is not None:
return bool(fe.platform["required"])
return fe.required
scripts/generate_pack.py
+66 -10
View File
@@ -36,6 +36,7 @@ import zipfile
from pathlib import Path, PurePosixPath
sys.path.insert(0, os.path.dirname(__file__))
from alternatives import core_file_spares, settle_sets, slot_set_keys
from common import resolve_platform_cores as _platform_cores
from common import (
ROMS_ROOT,
@@ -1295,8 +1296,13 @@ def generate_pack(
db, bios_dir, zip_contents, data_registry, validation_index,
validation_owners, offline, profiles=emu_profiles,
)
spares = core_file_spares(
core_files,
lambda fe: bool(region_drops)
and extra_region_key(fe, declared_root(config)) in region_drops,
) if required_only else set()
for fe in core_files:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in spares):
continue
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
@@ -1438,8 +1444,13 @@ def generate_pack(
if system_filter:
unfolded = _extras_for_systems(unfolded, system_filter)
prefix = _detect_extras_prefix(config, base_dest)
unfolded_spares = core_file_spares(
unfolded,
lambda fe: bool(region_drops)
and extra_region_key(fe, declared_root(config)) in region_drops,
) if required_only else set()
for fe in unfolded:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in unfolded_spares):
continue
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
continue
@@ -1780,6 +1791,16 @@ def generate_emulator_pack(
if archive:
archives.add(archive)
# What each destination came to, settled by set once the
# profile is read: a member another member fills is not missing,
# and a set nobody holds is reported once (alternatives.py). An
# archive unit is a member like a loose file.
outcomes: list[tuple[list[dict], bool, Callable[[], None] | None]] = []
spares = emulator_required_spares(
emu_name, profile, files, standalone, region_drops, db,
validation_index, zip_contents, data_registry,
) if required_only else set()
# Pack archives as units
archive_prefix = profile.get("archive_prefix", "")
for archive_name in sorted(archives):
@@ -1790,6 +1811,8 @@ def generate_emulator_pack(
own_entry = archive_entry(archive_name, files)
if required_only and archive_is_optional(own_entry, archive_name, files):
continue
if archive_dest in spares:
continue
local_path, status = resolve_file(
own_entry,
db,
@@ -1808,7 +1831,11 @@ def generate_emulator_pack(
if not (taken and local_path and _same_bytes(
local_path, packed_at.get(archive_dest.lower())
)):
report_missing(archive_name, archive_dest)
outcomes.append(([own_entry], False, functools.partial(
report_missing, archive_name, archive_dest
)))
else:
outcomes.append(([own_entry], True, None))
continue
if local_path and status not in ("not_found",):
if local_path.endswith(".zip"):
@@ -1822,8 +1849,11 @@ def generate_emulator_pack(
total_files += 1
if record is not None:
record.files.append(archive_dest)
outcomes.append(([own_entry], True, None))
else:
report_missing(archive_name, archive_dest)
outcomes.append(([own_entry], False, functools.partial(
report_missing, archive_name, archive_dest
)))
# Pack individual files (skip archive members). The entries of one
# profile on one destination are the releases the code accepts
@@ -1837,7 +1867,7 @@ def generate_emulator_pack(
if fe.get("archive"):
continue
dest = _resolve_destination(fe, standalone)
if not dest:
if not dest or dest in spares:
continue
if region_drops and dest in region_drops:
continue
@@ -1928,17 +1958,25 @@ def generate_emulator_pack(
held, packed_at.get(dest.lower())
):
# The file already at this path, asked for twice.
outcomes.append((alternatives, True, None))
continue
report_missing(alternatives[0]["name"], dest)
outcomes.append((alternatives, False, functools.partial(
report_missing, alternatives[0]["name"], dest
)))
continue
if chosen is None:
if refusal:
rejected_files.append(refusal)
outcomes.append((alternatives, False, None))
elif all(fe.get("unsourceable") for fe in alternatives):
unsourceable_files.append(alternatives[0]["name"])
outcomes.append((alternatives, False, functools.partial(
unsourceable_files.append, alternatives[0]["name"]
)))
else:
report_missing(alternatives[0]["name"], dest)
outcomes.append((alternatives, False, functools.partial(
report_missing, alternatives[0]["name"], dest
)))
continue
fe, local_path, how = chosen
@@ -1959,8 +1997,11 @@ def generate_emulator_pack(
total_files += 1
if record is not None:
record.files.append(dest)
outcomes.append((alternatives, True, None))
else:
missing_files.append(fe["name"])
outcomes.append((alternatives, False, functools.partial(
missing_files.append, fe["name"]
)))
finally:
if os.path.exists(tmp_path):
os.unlink(tmp_path)
@@ -1974,6 +2015,7 @@ def generate_emulator_pack(
# (but allow same file to be packed under different destinations,
# e.g., IPL.bin in GC/USA/ and GC/EUR/ from same source)
dedup_key_hash = f"{os.path.realpath(local_path)}:{dest}"
outcomes.append((alternatives, True, None))
if dedup_key_hash in seen_hashes:
continue
seen_hashes.add(dedup_key_hash)
@@ -1990,6 +2032,14 @@ def generate_emulator_pack(
if record is not None:
record.files.append(dest)
excused, _standing = settle_sets([
(slot_set_keys(emu_name, entries), entries[0], fills)
for entries, fills, _report in outcomes
])
for position, (_entries, _fills, report) in enumerate(outcomes):
if report is not None and position not in excused:
report()
# The trees after the files, as in a platform pack: a dump the
# profile declares wins over the copy a tree bundles at the same
# path (Dolphin's real dsp_rom.bin over Sys/GC's free one).
@@ -4379,8 +4429,13 @@ def _manifest_core_entries(
db, bios_dir, zip_contents, data_registry, validation_index or {},
owners, offline, profiles=profiles,
)
spares = core_file_spares(
core_files,
lambda fe: bool(region_drops)
and extra_region_key(fe, declared_root(config)) in region_drops,
) if required_only else set()
for fe in core_files:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in spares):
continue
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
if not dest:
@@ -5782,6 +5837,7 @@ from packextras import ( # noqa: E402,F401
extras_for_systems as _extras_for_systems,
platform_region_groups,
emulator_region_drops,
emulator_required_spares,
_emulator_region_group,
expand_agnostic_entries,
undeclared_report,
scripts/generate_site.py
+4 -4
View File
@@ -1116,7 +1116,7 @@ def _platform_export_rows(
),
"required": bool(entry.get("required", True)),
"region": _json_text(entry.get("region")),
"variant_group": _json_text(entry.get("variant_group")),
"selection_slot": _json_text(entry.get("selection_slot")),
"size": entry.get("size"),
"sha1": _json_text(entry.get("sha1")),
"sha256": _json_text(entry.get("sha256")),
@@ -1372,7 +1372,7 @@ def _write_sqlite_export(
CREATE TABLE platform_files (
platform_id TEXT NOT NULL, system TEXT NOT NULL, name TEXT NOT NULL,
destination TEXT NOT NULL, required INTEGER NOT NULL,
region TEXT, variant_group TEXT, size INTEGER,
region TEXT, selection_slot TEXT, size INTEGER,
sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT,
status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT,
zipped_file TEXT, hash_match_mandatory INTEGER, mandatory_declared INTEGER,
@@ -1455,7 +1455,7 @@ def _write_sqlite_export(
connection.executemany(
"INSERT INTO platform_files VALUES "
"(:platform_id, :system, :name, :destination, :required, :region, "
":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, "
":selection_slot, :size, :sha1, :sha256, :md5, :crc32, :status, :category, "
":collection_path, :resolved_sha1, :zipped_file, :hash_match_mandatory, "
":mandatory_declared, :curated, :shared_group, :mirror_of, :zip_digest)",
platform_files,
@@ -1699,7 +1699,7 @@ def generate_data_exports(
_csv_document(
[
"platform_id", "platform", "system", "name", "destination",
"required", "region", "variant_group", "size", "sha1",
"required", "region", "selection_slot", "size", "sha1",
"sha256", "md5", "crc32", "status", "category",
"collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS,
],
scripts/launchers.py
+1 -1
View File
@@ -1003,7 +1003,7 @@ def _launcher_entry(
entry = {
key: report.get(key)
for key in (
"profile", "emulator", "systems", "system", "region", "variant_group",
"profile", "emulator", "systems", "system", "region", "selection_slot",
"required", "hle_fallback", "category",
)
}
scripts/packextras.py
+66 -10
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
from collections.abc import Iterable
from common import PROFILE_IDENTITY_FIELDS, _norm_system_id, entry_system_ids
from alternatives import core_file_spares, required_only_spares, set_fields
from common import archive_members, holds_required_members
from common import declared_root, expand_platform_declared_names, platform_declarations
from common import load_emulator_profiles
@@ -403,8 +404,12 @@ def hidden_extras(
)
if system_ids:
extras = extras_for_systems(extras, system_ids)
spares = core_file_spares(
extras,
lambda fe: extra_region_key(fe, region_prefix) in (region_drops or set()),
) if required_only else set()
for fe in extras:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in spares):
continue
dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
if not dest or (region_drops and extra_region_key(fe, region_prefix) in region_drops):
@@ -657,7 +662,7 @@ def _agnostic_scan_extras(
"source_system": f.get("system"),
"source_systems": entry_system_ids(f, profile),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"selection_slot": f.get("selection_slot"),
"agnostic_scan": True,
}
extra["claim_order"] = (4, 0, emu_name)
@@ -813,7 +818,7 @@ def _companion_extras(
"source_system": seed.get("system"),
"source_systems": entry_system_ids(seed, profile),
"region": seed.get("region"),
"variant_group": seed.get("variant_group"),
"selection_slot": seed.get("selection_slot"),
"agnostic_scan": True,
}
extra["claim_order"] = (4, 0, emu_name)
@@ -987,7 +992,7 @@ def _archive_prefix_copies(
"source_system": archive_entry.get("system"),
"source_systems": entry_system_ids(archive_entry, profile),
"region": archive_entry.get("region"),
"variant_group": archive_entry.get("variant_group"),
"selection_slot": archive_entry.get("selection_slot"),
}
members = archive_members(archive_name, profile.get("files", []))
if members:
@@ -1392,7 +1397,8 @@ def _collect_emulator_extras(
"source_system": f.get("system"),
"source_systems": entry_system_ids(f, profile),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"selection_slot": f.get("selection_slot"),
**set_fields(f, profile),
}
# The same identity the first pass carries: without it the copy
# resolves on its name alone and a same-named file of another
@@ -1560,7 +1566,8 @@ def _report_extra(u: dict, name: str, dest: str) -> dict:
"source_system": u.get("system"),
"source_systems": u.get("systems", []),
"region": u.get("region"),
"variant_group": u.get("variant_group"),
"selection_slot": u.get("selection_slot"),
**{k: u[k] for k in ("required_groups", "set_declared") if k in u},
}
if u.get("members"):
extra["members"] = u["members"]
@@ -1837,9 +1844,9 @@ def extra_region_groups(
return (spellings or {}).get(_norm_system_id(sys_id), sys_id)
emulator = extra.get("source_emulator", "")
variant = extra.get("variant_group")
variant = extra.get("selection_slot")
if variant:
# A variant group is one slot of one core, whatever system each
# A selection slot is one slot of one core, whatever system each
# member is filed under: PicoDrive files its US Mega CD BIOS under
# sega-segacd and the EU/JP ones under sega-megacd, and find_bios
# picks one of the twelve. Keyed as emulator mode keys it.
@@ -1868,7 +1875,7 @@ def emulator_region_drops(
) -> set[str]:
"""Destinations an emulator-mode `--region` withdraws.
One group per system (or explicit variant_group) inside each profile, so
One group per system (or explicit selection_slot) inside each profile, so
a multi-system core such as O2EM keeps a fallback BIOS for every system
instead of letting one matching system empty another. The emulator pack
and `verify --emulator` both read this, keyed by pack destination.
@@ -1906,6 +1913,55 @@ def emulator_region_drops(
return region_mod.resolve_region_drops(region_groups, region_index, regions, held)
def emulator_required_spares(
emu_name: str,
profile: dict,
files: list[dict],
standalone: bool,
region_drops: set[str],
db: dict,
validation_index: dict,
zip_contents: dict | None,
data_registry: dict | None,
) -> set[str]:
"""The destinations an emulator --required-only pack leaves out: of each
required set, one held member stays, the first the code loads among
those the pack still carries after the region filter. An archive unit
is a member like a loose file: Kronos reads kronos/stvbios.zip, else
stvbios.zip. The pack and verify --emulator both read this."""
from alternatives import groups_of, rank_in, required_groups
from common import archive_entry
from packpaths import _archive_destination, _resolve_destination
candidates: list[dict] = []
archives: list[str] = []
for fe in files:
archive = fe.get("archive")
if archive and archive not in archives:
archives.append(archive)
own = archive_entry(archive, files)
if groups_of(own):
candidates.append({"dest": _archive_destination(
archive, profile.get("archive_prefix", ""),
profile.get("pack_structure"), standalone,
), "entry": own})
for fe in files:
if not fe.get("archive") and groups_of(fe):
candidates.append({"dest": _resolve_destination(fe, standalone), "entry": fe})
if not candidates:
return set()
spares = required_only_spares(
candidates,
owner=lambda _c: emu_name,
held=lambda c: _emulator_ships(
c["entry"], emu_name, profile, db, validation_index, zip_contents, data_registry,
),
gone=lambda c: c["dest"] in region_drops,
sets=lambda c: {g: rank_in(c["entry"], g) for g in required_groups(c["entry"], profile)},
)
return {c["dest"] for c in candidates if id(c) in spares}
def _emulator_ships(
file_entry: dict, emu_name: str, profile: dict, db: dict,
validation_index: dict, zip_contents: dict | None, data_registry: dict | None,
@@ -1946,7 +2002,7 @@ def _emulator_region_group(
Grouped together, geargrafx's LaserActive pac-n1.bin was dropped by
--emulator --region us and kept by --platform --region us.
"""
variant = file_entry.get("variant_group")
variant = file_entry.get("selection_slot")
if variant:
return f"{emu_name}:variant:{variant}"
system = file_entry.get("system")
scripts/slot.py
+42 -4
View File
@@ -29,6 +29,8 @@ rank: the second core reads it by its own rule, which no rank expresses.
from __future__ import annotations
from alternatives import set_keys
BIOS_CATEGORY = "bios"
@@ -65,9 +67,11 @@ def build_slot_index(profiles: dict) -> dict[str, dict]:
entry = index.setdefault(
key,
{"rank": None, "conflict": False, "regions": set(),
"emulators": [], "ranked_by": set(), "unranked_by": set()},
"emulators": [], "ranked_by": set(), "unranked_by": set(),
"sets": set()},
)
entry["regions"] |= {str(r) for r in (f.get("region") or [])}
entry["sets"] |= set(set_keys(emu_name, f))
rank = f.get("priority")
entry["unranked_by" if rank is None else "ranked_by"].add(emu_name)
if rank is not None:
@@ -123,7 +127,7 @@ def resolve_slot_drops(
# A slot is a system AND a declared region: the Japanese and American
# PlayStation BIOS are not alternatives to each other, and comparing their
# ranks across regions would pick one territory's file for another's.
tiers: dict[tuple[str, tuple[str, ...]], list[tuple[str, int | None]]] = {}
bucketed: dict[tuple[str, tuple[str, ...]], list[tuple[str, int | None, frozenset]]] = {}
for group_id, members in groups.items():
for destination, name in members:
if held is not None and destination not in held:
@@ -134,9 +138,18 @@ def resolve_slot_drops(
continue
tier = tuple(sorted(entry["regions"]))
rank = None if entry.get("conflict") else entry["rank"]
tiers.setdefault((group_id, tier), []).append((destination, rank))
bucketed.setdefault((group_id, tier), []).append(
(destination, rank, frozenset(entry.get("sets") or ()))
)
# A rank orders the members of one set: px68k-sdl ranks its IPL and its
# font ROM lists each from 1, and the two never compete. Candidates are
# compared when they share a set, and those in no set among themselves.
tiers: dict[tuple[str, tuple[str, ...], int], list[tuple[str, int | None]]] = {}
for (group_id, tier), found in bucketed.items():
for part, component in enumerate(_by_shared_set(found)):
tiers[(group_id, tier, part)] = component
for (group_id, tier), candidates in tiers.items():
for (group_id, tier, _part), candidates in tiers.items():
if len(candidates) < 2:
keep.update(dest for dest, _p in candidates)
continue
@@ -160,3 +173,28 @@ def resolve_slot_drops(
drop.update(dest for dest, _p in candidates if dest != winners[0])
return drop - keep, sorted(undecidable)
def _by_shared_set(
found: list[tuple[str, int | None, frozenset]],
) -> list[list[tuple[str, int | None]]]:
"""The candidates of one tier split where no set links them."""
parent = list(range(len(found)))
def root(i: int) -> int:
while parent[i] != i:
parent[i] = parent[parent[i]]
i = parent[i]
return i
first_of: dict[object, int] = {}
for i, (_dest, _rank, sets) in enumerate(found):
for key in sets or {None}:
if key in first_of:
parent[root(i)] = root(first_of[key])
else:
first_of[key] = i
parts: dict[int, list[tuple[str, int | None]]] = {}
for i, (dest, rank, _sets) in enumerate(found):
parts.setdefault(root(i), []).append((dest, rank))
return list(parts.values())
scripts/truth.py
+32 -3
View File
@@ -18,6 +18,7 @@ from common import (
resolve_platform_cores,
standalone_set as platform_standalone_set,
)
from alternatives import groups_of
from hashing import compute_hashes
from validation import (
filter_files_by_mode,
@@ -276,11 +277,15 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None:
existing["_source_refs"] = existing.get("_source_refs", set()) | {sr_key}
else:
existing.setdefault("_source_refs", set())
if file_entry.get("required"):
if groups_of(file_entry):
_fold_set_member(existing, file_entry, emu_name)
elif file_entry.get("required"):
existing["required"] = True
# Which cores need it: `required` above is the union, true when
# any core needs it, and a package of one core must not read it.
existing["_required_by"] = existing.get("_required_by", set()) | {emu_name}
elif file_entry.get("required") is not None:
existing.setdefault("required", file_entry["required"])
_merge_hashes(existing, file_entry, emu_name, revision)
# Merge non-hash data fields if existing lacks them.
# A core that creates an entry without size/path/validation may be
@@ -292,6 +297,21 @@ def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None:
_merge_region(existing, file_entry.get("region"))
def _fold_set_member(entry: dict, file_entry: dict, emu_name: str) -> None:
"""A core that takes this file as one of a set needs the set, not it.
Yabause loads the first of saturn_bios.bin, sega_101.bin and
mpr-17933.bin it finds. None of the three is required on its own, so
the core stays out of `required` and `_required_by`, and `_one_of`
names it with `one_of_required` saying whether its set is required.
"""
entry["_one_of"] = entry.get("_one_of", set()) | {emu_name}
if file_entry.get("required", True):
entry["one_of_required"] = True
else:
entry.setdefault("one_of_required", False)
def _merge_region(existing: dict, theirs: str | list | None) -> None:
"""The regions every reader selects the file for, the pack's own rule.
@@ -379,7 +399,8 @@ def _merge_priority(existing: dict, theirs: int | None) -> None:
def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
entry: dict = {"name": file_entry["name"]}
if file_entry.get("required") is not None:
grouped = bool(groups_of(file_entry))
if file_entry.get("required") is not None and not grouped:
entry["required"] = file_entry["required"]
for field in _COPIED_FIELDS:
val = file_entry.get(field)
@@ -396,7 +417,9 @@ def _new_truth_entry(file_entry: dict, emu_name: str, db: dict | None) -> dict:
if not file_entry.get("region"):
entry["_region_void"] = True
entry["_cores"] = {emu_name}
entry["_required_by"] = {emu_name} if file_entry.get("required") else set()
entry["_required_by"] = {emu_name} if file_entry.get("required") and not grouped else set()
if grouped:
_fold_set_member(entry, file_entry, emu_name)
entry["_named_by"] = {emu_name} if file_entry.get("_opens_by_name", True) else set()
entry["_brought_by"] = _brought(file_entry, emu_name)
sr = file_entry.get("source_ref")
@@ -850,6 +873,9 @@ def generate_platform_truth(
fe.pop("_region_void", None)
fe["_cores"] = sorted(fe.get("_cores", set()))
fe["_required_by"] = sorted(fe.get("_required_by", set()))
one_of = fe.pop("_one_of", None)
if one_of:
fe["one_of"] = sorted(one_of)
fe["_named_by"] = sorted(fe.get("_named_by", set()))
fe["_brought_by"] = sorted(fe.get("_brought_by", set()))
if not fe["_brought_by"]:
@@ -952,6 +978,9 @@ def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
hash_mismatch.append(mismatch)
t_req = t_entry.get("required")
s_req = s_entry.get("required")
if t_entry.get("one_of_required") and not t_req:
# Needed as one of a set: no value of its own to compare.
t_req = None
if t_req is not None and s_req is not None and t_req != s_req:
required_mismatch.append(
{
scripts/validate_schemas.py
+31
View File
@@ -86,6 +86,7 @@ def _validate_yaml_directory(
out.extend(_save_dir_without_library_name(path, data))
out.extend(_systems_outside_profile(path, data))
out.extend(_doubled_paths(path))
out.extend(_group_requirement_split(path, data))
return out
@@ -122,6 +123,36 @@ def _systems_outside_profile(path: Path, document: object) -> list[str]:
return out
def _group_requirement_split(path: Path, document: object) -> list[str]:
"""A set named by a member and not declared, declared and never used,
or a member that writes a required of its own.
The requirement belongs to the set, in the profile's groups map, and a
member's is derived from it. A value on the member is a second source
that can contradict the first: a true on the first name only wrote the
search order into required, and the other names read as optional.
"""
from alternatives import groups_of
if not isinstance(document, dict):
return []
label = path.relative_to(ROOT)
declared = set((document.get("groups") or {}).keys())
used: set[str] = set()
out: list[str] = []
for index, entry in enumerate(document.get("files") or []):
if not isinstance(entry, dict):
continue
names = groups_of(entry)
used.update(names)
if names and "required" in entry:
out.append(f"{label}: files[{index}] ({entry.get('name')}) is in a group "
"and writes required: the set's requirement is in groups")
out.extend(f"{label}: group {g} has no entry in groups" for g in sorted(used - declared))
out.extend(f"{label}: groups declares {g}, which no entry names" for g in sorted(declared - used))
return out
def _checks_without_values(path: Path, document: object) -> list[str]:
"""A check an entry declares with nothing to compare against.
scripts/verify.py
+174 -39
View File
@@ -37,6 +37,14 @@ import slots
sys.path.insert(0, os.path.dirname(__file__))
from hashing import parse_md5_list
from alternatives import (
core_file_spares,
set_fields,
set_keys,
set_requirements,
settle_sets,
slot_set_keys,
)
from common import (
_foreign_tree,
_owner_tokens,
@@ -537,6 +545,9 @@ def find_undeclared_files(
config, profiles, relevant, db, declared_destinations(config)
)
launcher_candidates: list[tuple[str, bool, dict, dict]] = []
# Each grouped requirement met, as (set keys, entry, fills, (reported
# entry, the entry it is a claim on)): settled once every core is read.
set_records: list[tuple[tuple, dict, bool, tuple[dict, dict | None] | None]] = []
undeclared = []
seen_files: set[tuple] = set()
# Key -> the entry reported for it. A later profile reading the same
@@ -590,7 +601,7 @@ def find_undeclared_files(
f.get("archive"),
dest,
f.get("system"),
f.get("variant_group"),
f.get("selection_slot"),
region_key,
# Below the save directory each core is handed a folder
# of its own: two cores reading hiscore.dat there read
@@ -615,6 +626,9 @@ def find_undeclared_files(
)
if verdict == "settled":
seen_files.add(seen_key)
# Answered by the platform's own declaration, which its
# report covers: the set it belongs to is spoken for.
set_records.append((set_keys((emu_name, is_standalone), f), f, True, None))
continue
if verdict == "skip":
continue
@@ -627,7 +641,7 @@ def find_undeclared_files(
archive,
f.get("system_dir"),
f.get("system"),
f.get("variant_group"),
f.get("selection_slot"),
region_key,
is_standalone,
)
@@ -642,7 +656,7 @@ def find_undeclared_files(
"systems": entry_system_ids(f, profile),
"system": f.get("system"),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"selection_slot": f.get("selection_slot"),
"name": archive,
"archive": archive,
"path": system_dir_path(f, archive),
@@ -660,6 +674,14 @@ def find_undeclared_files(
}
if f.get("system_dir"):
archive_entries[archive_key]["system_dir"] = f["system_dir"]
# The archive's own declaration names its sets: an
# archive is a member like a loose file.
own = archive_entry(archive, profile.get("files", []))
archive_entries[archive_key].update(set_fields(own, profile))
set_records.append((
set_keys((emu_name, is_standalone), own), own, in_repo,
(archive_entries[archive_key], None),
))
entry = archive_entries[archive_key]
entry["archive_file_count"] += 1
if f.get("required", False):
@@ -740,7 +762,7 @@ def find_undeclared_files(
"systems": entry_system_ids(f, profile),
"system": f.get("system"),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"selection_slot": f.get("selection_slot"),
"name": fname,
"path": dest,
"required": f.get("required", False),
@@ -750,6 +772,7 @@ def find_undeclared_files(
"in_repo": in_repo,
"note": f.get("note", ""),
"checks": sorted(checks) if checks else [],
**set_fields(f, profile),
"source_ref": f.get("source_ref"),
"expected": _build_expected(f, checks),
**_identity_of(f),
@@ -777,12 +800,16 @@ def find_undeclared_files(
entry["root"] = "roms"
elif launcher_layer and not save_root:
launcher_candidates.append((emu_name, is_standalone, f, entry))
keys = set_keys((emu_name, is_standalone), f)
if duplicate:
first.setdefault("other_claims", []).append(entry)
set_records.append((keys, f, in_repo, (entry, first)))
continue
reported[seen_key] = entry
undeclared.append(entry)
set_records.append((keys, f, in_repo, (entry, None)))
settled: set[int] = set()
if launcher_layer:
added, settled = launcher_layer.settle(
launcher_candidates, lambda report: held_record(report, db)
@@ -793,7 +820,70 @@ def find_undeclared_files(
for entry in sorted(archive_entries.values(), key=lambda e: e["name"]):
undeclared.append(entry)
return undeclared
return _settle_core_sets(undeclared, set_records, settled, profiles)
def _settle_core_sets(
undeclared: list[dict],
records: list[tuple[tuple, dict, bool, tuple[dict, dict | None] | None]],
launcher_settled: set[int],
profiles: dict[str, dict],
) -> list[dict]:
"""The core complement with each set of alternatives settled.
A member the collection lacks is dropped when another member is held,
declared by the platform or placed by its launcher, and a set nobody
holds keeps one entry, its first member, naming the others. A claim
another profile shares stays: its own core may need that very file.
"""
# Archive units rank first among equals, as the builder orders them.
grouped = sorted(
(r for r in records if r[0]),
key=lambda r: not (r[3] is not None and r[3][0].get("archive")),
)
if not grouped:
return undeclared
excused, standing = settle_sets([
(keys, f, fills or (ref is not None and id(ref[0]) in launcher_settled))
for keys, f, fills, ref in grouped
], lambda key: set_requirements(profiles.get(key[0][0], {})).get(key[1], True))
declared_sets = {
key for keys, _f, _fills, ref in grouped if ref is None for key in keys
}
dropped: set[int] = set()
for position, (keys, _f, _fills, ref) in enumerate(grouped):
if ref is None:
continue
entry, host = ref
required_keys = [k for k in keys if k[1] in entry.get("required_groups", ())]
if required_keys and all(k in declared_sets for k in required_keys):
# The platform declares a member of each of its required sets:
# under --required-only that declaration carries them.
entry["set_declared"] = True
if position in standing:
entry["alternatives"] = standing[position].others
entry["required"] = standing[position].required
if position not in excused:
continue
if host is not None:
host["other_claims"] = [c for c in host.get("other_claims", []) if c is not entry]
if not host["other_claims"]:
del host["other_claims"]
else:
dropped.add(id(entry))
out: list[dict] = []
for entry in undeclared:
if id(entry) not in dropped:
out.append(entry)
continue
# Another core reading the same requirement still needs it.
claims = entry.get("other_claims") or []
if claims:
promoted = dict(claims[0])
if claims[1:]:
promoted["other_claims"] = claims[1:]
out.append(promoted)
return out
def find_exclusion_notes(
@@ -1510,6 +1600,10 @@ def verify_platform(
)
not in region_drops
]
if required_only:
# One member of a required set, as the builder keeps it.
spares = core_file_spares(undeclared)
undeclared = [u for u in undeclared if id(u) not in spares]
_mark_unplaced(config, undeclared, profiles)
from packextras import ExtraResolver
@@ -1826,6 +1920,8 @@ def _print_undeclared_entry(
name_label = f"{name} ({arc_count} file{'s' if arc_count != 1 else ''})"
else:
name_label = name
if u.get("alternatives") and not u.get("in_repo"):
name_label += f" (or {', '.join(u['alternatives'])})"
print(f" {prefix}: {u['emulator']} needs {name_label}")
checks = u.get("checks", [])
if checks:
@@ -2200,6 +2296,9 @@ def verify_emulator(
return local_path, status
details = []
# Members of a set another member fills, or that the set's first member
# stands for: not missing, and out of every count.
set_members: list[dict] = []
file_status: dict[str, str] = {}
file_severity: dict[str, str] = {}
dest_to_name: dict[str, str] = {}
@@ -2305,6 +2404,18 @@ def verify_emulator(
if fe.get("archive")
or _resolve_destination(fe, standalone) not in region_drops
]
spares: set[str] = set()
if required_only:
from packextras import emulator_required_spares
spares = emulator_required_spares(
emu_name, profile, files, standalone, region_drops, db,
validation_index, zip_contents, data_registry,
)
files = [
fe for fe in files
if fe.get("archive") or _resolve_destination(fe, standalone) not in spares
]
# Check data directories (only notice if not cached)
for dd in deliverable(
@@ -2335,6 +2446,12 @@ def verify_emulator(
occupy_trees(emu_name, profile)
continue
# Each destination's outcome, settled by set once the profile is
# read: an archive unit is a member like a loose file (Kronos reads
# kronos/stvbios.zip, else stvbios.zip).
placed_results: list[tuple[str, dict, bool, bool, list[dict]]] = []
fallback_reason = "declared hash contradicted by the local file"
# Verify archives as units (e.g., neogeo.zip, aes.zip)
seen_archives: set[str] = set()
for file_entry in files:
@@ -2342,6 +2459,11 @@ def verify_emulator(
if archive and archive not in seen_archives:
seen_archives.add(archive)
own_entry = archive_entry(archive, files)
if _archive_destination(
archive, profile.get("archive_prefix", ""),
profile.get("pack_structure"), standalone,
) in spares:
continue
local_path, resolve_status = pending(own_entry, *resolve_file(
own_entry,
db,
@@ -2384,7 +2506,6 @@ def verify_emulator(
result["system"] = file_entry.get("system", "")
result["hle_fallback"] = False
result["ground_truth"] = build_ground_truth(archive, validation_index)
details.append(result)
# Where the pack puts it: an archive_prefix or pack_structure
# moves it, and a loose same-named entry is another file.
dest = _archive_destination(
@@ -2408,25 +2529,13 @@ def verify_emulator(
elif local_path:
occupy(dest, local_path, emu_name)
packed_dests.add(dest)
dest_to_name[dest] = archive
cur = result["status"]
prev = file_status.get(dest)
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(
prev, 0
):
file_status[dest] = cur
sev = _emulator_severity(cur, required, False)
prev_sev = file_severity.get(dest)
if prev_sev is None or _SEVERITY_ORDER.get(
sev, 0
) > _SEVERITY_ORDER.get(prev_sev, 0):
file_severity[dest] = sev
placed_results.append((dest, result, required, False, [own_entry]))
# Results of this emulator, by destination. Several entries of one
# profile on one destination are the releases the code accepts
# there (a shareware and a retail HOG, each GRP in a size table):
# holding one of them fills the slot, and the others are not gaps.
emu_results: dict[str, list[tuple[dict, bool, bool]]] = {}
emu_results: dict[str, list[tuple[dict, bool, bool, dict]]] = {}
for file_entry in files:
# Skip archive members (verified as archive units above)
@@ -2526,13 +2635,13 @@ def verify_emulator(
# The slot the emulator pack places this entry in: standalone_path
# under --standalone, the sanitised path.
dest = _resolve_destination(file_entry, standalone) or name
emu_results.setdefault(dest, []).append((result, required, hle))
emu_results.setdefault(dest, []).append((result, required, hle, file_entry))
for dest, alternatives in emu_results.items():
# What the emulator pack places here: an accepted file, else a
# file whose hash the profile contradicts. A refused copy is held
# but never packed.
fallback_reason = "declared hash contradicted by the local file"
slot_entries = [a[3] for a in alternatives]
if len(alternatives) > 1:
# Reduced in the order generate_emulator_pack chooses, so the
# destination's status is the one the pack reports: a slot it
@@ -2585,27 +2694,50 @@ def verify_emulator(
elif not block and packed is not None:
occupy(dest, packed, emu_name)
packed_dests.add(dest)
for result, required, hle in alternatives:
for result, required, hle, file_entry in alternatives:
if block_reason and result["status"] != Status.MISSING:
result["status"] = Status.MISSING
result["reason"] = block_reason
result["destination"] = dest
details.append(result)
placed_results.append((dest, result, required, hle, slot_entries))
# Aggregate by destination (path if available, else name)
dest_to_name[dest] = result["name"]
cur = result["status"]
prev = file_status.get(dest)
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(
prev, 0
):
file_status[dest] = cur
sev = _emulator_severity(cur, required, hle)
prev_sev = file_severity.get(dest)
if prev_sev is None or _SEVERITY_ORDER.get(
sev, 0
) > _SEVERITY_ORDER.get(prev_sev, 0):
file_severity[dest] = sev
# A set the code takes one member of is one need: once a member is
# in the pack the others are not missing, and a set nobody holds is
# one gap under its first member. Settled per profile, before the
# destinations of several profiles are merged.
excused, standing = settle_sets([
(
slot_set_keys(emu_name, entries),
entries[0],
result["status"] == Status.OK
or (result["status"] == Status.UNTESTED
and result.get("reason") == fallback_reason),
)
for _dest, result, _required, _hle, entries in placed_results
], lambda key: set_requirements(profile).get(key[1], True))
for position, (dest, result, required, hle, _fe) in enumerate(placed_results):
if position in excused and result["status"] == Status.MISSING:
set_members.append(result)
continue
if position in standing:
result["alternatives"] = standing[position].others
result["required"] = required = standing[position].required
details.append(result)
# Aggregate by destination (path if available, else name)
dest_to_name[dest] = result["name"]
cur = result["status"]
prev = file_status.get(dest)
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(
prev, 0
):
file_status[dest] = cur
sev = _emulator_severity(cur, required, hle)
prev_sev = file_severity.get(dest)
if prev_sev is None or _SEVERITY_ORDER.get(
sev, 0
) > _SEVERITY_ORDER.get(prev_sev, 0):
file_severity[dest] = sev
occupy_trees(emu_name, profile)
@@ -2639,6 +2771,7 @@ def verify_emulator(
"severity_counts": counts,
"status_counts": status_counts,
"details": details,
"set_members": set_members,
"data_dir_notices": sorted(set(data_dir_notices)),
"ground_truth_coverage": {
"with_validation": with_validation,
@@ -2744,7 +2877,9 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
seen.add(d["name"])
req = "required" if d.get("required", True) else "optional"
hle = ", HLE available" if d.get("hle_fallback") else ""
print(f" MISSING ({req}{hle}): {d['name']}")
others = d.get("alternatives")
either = f" (or {', '.join(others)})" if others else ""
print(f" MISSING ({req}{hle}): {d['name']}{either}")
gt = d.get("ground_truth", [])
if gt:
if verbose: