feat: give each set its requirement and one pick

Three changes to the search-list model, one commit because the readers
and the profiles must move together for the schema to hold.

Set requirement in a groups map. pom2 falls back on apple2.rom for the
][ and the ][+, and on apple2e.rom for the //e it boots and for the
unenhanced //e: one required on the file could not say both. A profile
now declares groups: {name: {required}}, a member writes no required
of its own (schema and validate_schemas refuse one) and load_emulator_
profiles derives it, true when one of its sets is. group takes a name,
a list, or a map from set to rank where the rank differs per set. pom2
is grouped from SystemProfile.cpp and MainWindow.cpp:246-296 at its pin.

One member per required set under --required-only. The pick is the
first the code reads among the held members the region filter leaves,
by rank then declaration order; a set the platform declares a member
of is carried by that declaration. The emulator pack, verify
--emulator, the platform pack, its manifest and verify --platform read
alternatives.required_only_spares or core_file_spares.

variant_group is now selection_slot: files a core chooses between by
a rule other than first found (disc territory for PicoDrive and
Pizza Boy SC, the o2em_bios option, a WebAssembly probe in SWF
Player). Where the code takes any one, the field became a group:
ARMSX1 (ArmSxBackend.java:149-157, name order), ePSXe (load_bios
0x433540-0x433621 of the 2.0.5 Linux x64 binary, one configured path,
HLE when BiosHLE is set), FPse and FPse-NG, DXX-Rebirth's HOGs
(inferno.cpp:472-512). odin and zesarux carried it with no effect.
slot.py compares ranks only between candidates that share a set.
This commit is contained in:
Abdessamad Derraz committed 2026-11-09 18:19:32 +01:00
1 parent 1e12d9fe38
commit 315dab82b7
38 files changed
+661 -432

No files matched your search

+142 -22
View File
@@ -3,32 +3,88 @@
A profile writes ``group:`` on the entries of one search list. Yabause tries
saturn_bios.bin, sega_101.bin, then mpr-17933.bin and loads the first it
finds (libretro.c:1035-1049): it needs one of them and none in particular.
``required`` on a member states the set's requirement, and ``priority`` keeps
the order the code reads them in.
The requirement belongs to the set, in the profile's ``groups:`` map, and a
member's ``required`` is derived from the sets it is in: true when one of
them is required. A fallback two machine models share (pom2's apple2.rom)
then needs no value of its own. ``priority`` keeps the order the code reads
the members in.
Holding one member fills the set, so the other members are not missing. A
set nobody holds is one gap, reported under its first member, never one gap
per member. Every reader settles sets through ``settle_sets`` so that verify,
the gap report and the packs agree on what is missing.
set nobody holds is one gap, reported under its first member with the set's
requirement, never one gap per member. A --required-only build keeps one
member of a required set, the first the code would load among those held.
Every reader goes through this module so that verify, the gap report, the
packs and the manifests agree.
"""
from __future__ import annotations
from collections.abc import Hashable, Iterable, Sequence
from collections.abc import Callable, Hashable, Iterable, Sequence
from dataclasses import dataclass
_UNRANKED = 1 << 30
def groups_of(entry: dict) -> tuple[str, ...]:
"""The sets an entry belongs to: one name, or a list for a file two
machine models both fall back on."""
"""The sets an entry belongs to: one name, a list for a file two machine
models both fall back on, or a map from set to the entry's rank there."""
value = entry.get("group")
if not value:
return ()
values = value if isinstance(value, list) else [value]
values = value if isinstance(value, (list, dict)) else [value]
return tuple(str(v) for v in values if v)
def rank_in(entry: dict, group: str) -> int | None:
"""The rank of an entry in one of its sets: the map's value when the
entry ranks differently in each (pom2's apple2.rom is second for the ][
and third at boot), else its priority."""
value = entry.get("group")
if isinstance(value, dict) and isinstance(value.get(group), int):
return value[group]
priority = entry.get("priority")
return priority if isinstance(priority, int) else None
def set_requirements(profile: dict) -> dict[str, bool]:
"""The requirement of each set the profile declares."""
declared = profile.get("groups") or {}
return {
str(name): bool((spec or {}).get("required", True))
for name, spec in declared.items()
}
def derive_member_requirements(profile: dict) -> None:
"""Write on each member the requirement its sets give it.
Read once when a profile is loaded, so every reader sees ``required``
as it always has. A member is required when one of its sets is.
"""
requirements = set_requirements(profile)
for entry in profile.get("files") or []:
if not isinstance(entry, dict):
continue
names = groups_of(entry)
if names and "required" not in entry:
entry["required"] = any(requirements.get(n, True) for n in names)
def required_groups(entry: dict, profile: dict) -> list[str]:
"""The required sets of its profile an entry is a member of."""
requirements = set_requirements(profile)
return [n for n in groups_of(entry) if requirements.get(n, True)]
def set_fields(entry: dict, profile: dict) -> dict:
"""What a --required-only build reads on a core file to keep one member
of a set: the entry's required sets, each with its rank there."""
sets = required_groups(entry, profile)
if not sets:
return {}
return {"required_groups": {group: rank_in(entry, group) for group in sets}}
def set_keys(owner: Hashable, entry: dict) -> tuple[tuple, ...]:
"""The keys of an entry's sets, scoped to its owner (a profile, or a
profile under one build)."""
@@ -45,39 +101,48 @@ def slot_set_keys(owner: Hashable, entries: Iterable[dict]) -> tuple[tuple, ...]
return tuple(keys)
def _rank(entry: dict, position: int) -> tuple[int, int]:
priority = entry.get("priority")
return (priority if isinstance(priority, int) else _UNRANKED, position)
def _rank(rank: int | None, position: int) -> tuple[int, int]:
return (rank if isinstance(rank, int) else _UNRANKED, position)
@dataclass
class Standing:
"""The member kept as the one gap of the sets nobody holds."""
others: list[str]
required: bool
def settle_sets(
members: Sequence[tuple[tuple[tuple, ...], dict, bool]],
) -> tuple[set[int], dict[int, list[str]]]:
set_required: Callable[[tuple], bool] = lambda _key: True,
) -> tuple[set[int], dict[int, Standing]]:
"""Which unheld members are excused, and which one stands for its set.
``members`` lists ``(set keys, entry, fills)`` in the order the profile
declares them, where ``fills`` says the member is held (or answered by a
declaration that reports it). Returns the positions to leave out of a
report and, for each set no member fills, the position kept as its one
gap with the names of the others. A member fills or not on its own: a
held one is never excused, and an unheld one stays reported when it
stands for at least one unfilled set.
gap with the names of the others and the set's requirement. A member
fills or not on its own: a held one is never excused, and an unheld one
stays reported when it stands for at least one unfilled set.
"""
by_set: dict[tuple, list[int]] = {}
for position, (keys, _entry, _fills) in enumerate(members):
for key in keys:
by_set.setdefault(key, []).append(position)
standing: dict[int, list[str]] = {}
for positions in by_set.values():
standing: dict[int, Standing] = {}
for key, positions in by_set.items():
if any(members[p][2] for p in positions):
continue
first = min(positions, key=lambda p: _rank(members[p][1], p))
names = standing.setdefault(first, [])
first = min(positions, key=lambda p: _rank(rank_in(members[p][1], key[1]), p))
held_up = standing.setdefault(first, Standing([], False))
held_up.required = held_up.required or set_required(key)
for p in positions:
name = str(members[p][1].get("name", ""))
if p != first and name not in names:
names.append(name)
if p != first and name not in held_up.others:
held_up.others.append(name)
excused = {
position
@@ -85,3 +150,58 @@ def settle_sets(
if keys and not fills and position not in standing
}
return excused, standing
def required_only_spares(
entries: Sequence[dict],
owner: Callable[[dict], Hashable],
held: Callable[[dict], bool],
gone: Callable[[dict], bool] = lambda _entry: False,
sets: Callable[[dict], dict[str, int | None]] = lambda e: e.get("required_groups") or {},
) -> set[int]:
"""The ids of the held members a --required-only build leaves out.
``sets`` maps the required sets of an entry to its rank in each: a core
file of a platform pack carries them as ``required_groups``.
A required set needs one file: the build keeps the member the code
would load first among those it holds and still carries (``gone`` says
a region filter took it), by ``priority`` then declaration order, and
drops the others. A member of an optional set is not touched here,
nor one that another required set of its own still needs.
"""
candidates: dict[tuple, list[tuple[tuple[int, int], int]]] = {}
for position, entry in enumerate(entries):
if not held(entry) or gone(entry):
continue
for group, rank in sets(entry).items():
candidates.setdefault((owner(entry), group), []).append(
(_rank(rank, position), position)
)
kept: set[int] = set()
members: set[int] = set()
for ranked in candidates.values():
members.update(position for _rank_key, position in ranked)
kept.add(min(ranked)[1])
return {id(entries[p]) for p in members - kept}
def core_file_spares(
entries: Sequence[dict], gone: Callable[[dict], bool] = lambda _entry: False,
) -> set[int]:
"""The core files of a platform pack a --required-only build leaves out.
A set the platform declares a member of is carried by that declaration,
so its other members are spares. Otherwise one held member per required
set stays, as in an emulator pack. Read by the builder, the manifest
and verify alike.
"""
answered = {id(e) for e in entries if e.get("set_declared")}
rest = [e for e in entries if id(e) not in answered]
return answered | required_only_spares(
rest,
owner=lambda e: e.get("source_profile") or e.get("profile") or "",
held=lambda e: e.get("in_repo", True) is not False,
gone=gone,
)
+3
View File
@@ -22,6 +22,8 @@ import zipfile
from pathlib import Path
from typing import Any, TextIO
from alternatives import derive_member_requirements
# Every build script imports this module. verify.py, slots.py, validate_pr.py
# and cross_reference.py started under 3.8 and stopped halfway on a
# str.removeprefix AttributeError, which named no version.
@@ -2346,6 +2348,7 @@ def load_emulator_profiles(
if "emulator" not in profile:
continue
_normalize_hash_fields(profile)
derive_member_requirements(profile)
profiles[f.stem] = profile
_emulator_profiles_cache[cache_key] = profiles
return profiles
+10 -5
View File
@@ -23,7 +23,7 @@ from collections import Counter
from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__))
from alternatives import set_keys, settle_sets
from alternatives import set_keys, set_requirements, settle_sets
from common import (
platform_arg,
ProfileSelectionError,
@@ -697,7 +697,7 @@ def slot_key(f: dict, standalone: bool = False) -> tuple:
f.get("archive"),
system_dir_path(f, where or fname),
f.get("system"),
f.get("variant_group"),
f.get("selection_slot"),
f.get("mode", "both"),
read_from_save_dir(f, standalone),
)
@@ -886,7 +886,7 @@ def _cross_reference_profile(
archive_key = (
archive,
f.get("system"),
f.get("variant_group"),
f.get("selection_slot"),
f.get("mode", "both"),
)
if archive_key not in archive_gaps:
@@ -969,12 +969,17 @@ def _cross_reference_profile(
# nobody holds is one gap under its first member (alternatives.py), as
# verify and the packs count it.
grouped = [r for r in set_records if r[0]]
excused, standing = settle_sets([(keys, f, fills) for keys, f, fills, _l, _r in grouped])
requirements = set_requirements(profile)
excused, standing = settle_sets(
[(keys, f, fills) for keys, f, fills, _l, _r in grouped],
lambda key: requirements.get(key[1], True),
)
for at, (_keys, member, _fills, rows, set_row) in enumerate(grouped):
if set_row is None or rows is None:
continue
if at in standing:
set_row["alternatives"] = standing[at]
set_row["alternatives"] = standing[at].others
set_row["required"] = standing[at].required
if at in excused:
rows.remove(set_row)
fates[position[id(member)]] = "other_versions"
+25 -5
View File
@@ -36,7 +36,7 @@ import zipfile
from pathlib import Path, PurePosixPath
sys.path.insert(0, os.path.dirname(__file__))
from alternatives import settle_sets, slot_set_keys
from alternatives import core_file_spares, settle_sets, slot_set_keys
from common import resolve_platform_cores as _platform_cores
from common import (
installed_data_refs,
@@ -1295,8 +1295,13 @@ def generate_pack(
db, bios_dir, zip_contents, data_registry, validation_index,
validation_owners, offline, profiles=emu_profiles,
)
spares = core_file_spares(
core_files,
lambda fe: bool(region_drops)
and extra_region_key(fe, declared_root(config)) in region_drops,
) if required_only else set()
for fe in core_files:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in spares):
continue
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
@@ -1434,8 +1439,13 @@ def generate_pack(
if system_filter:
unfolded = _extras_for_systems(unfolded, system_filter)
prefix = _detect_extras_prefix(config, base_dest)
unfolded_spares = core_file_spares(
unfolded,
lambda fe: bool(region_drops)
and extra_region_key(fe, declared_root(config)) in region_drops,
) if required_only else set()
for fe in unfolded:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in unfolded_spares):
continue
if region_drops and extra_region_key(fe, declared_root(config)) in region_drops:
continue
@@ -1826,9 +1836,13 @@ def generate_emulator_pack(
# there (raze's seven SW.GRP): the first one the core accepts
# fills the slot, as verify --emulator credits it, and the slot
# has one outcome however many alternatives it lists.
spares = emulator_required_spares(
emu_name, profile, files, standalone, region_drops, db,
validation_index, zip_contents, data_registry,
) if required_only else set()
slots: dict[str, list[dict]] = {}
for fe in files:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in spares):
continue
if fe.get("archive"):
continue
@@ -4398,8 +4412,13 @@ def _manifest_core_entries(
db, bios_dir, zip_contents, data_registry, validation_index or {},
owners, offline, profiles=profiles,
)
spares = core_file_spares(
core_files,
lambda fe: bool(region_drops)
and extra_region_key(fe, declared_root(config)) in region_drops,
) if required_only else set()
for fe in core_files:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in spares):
continue
dest = sanitize_pack_path(fe.get("destination", fe["name"]))
if not dest:
@@ -5770,6 +5789,7 @@ from packextras import ( # noqa: E402,F401
extras_for_systems as _extras_for_systems,
platform_region_groups,
emulator_region_drops,
emulator_required_spares,
_emulator_region_group,
expand_agnostic_entries,
undeclared_report,
+4 -4
View File
@@ -1115,7 +1115,7 @@ def _platform_export_rows(
),
"required": bool(entry.get("required", True)),
"region": _json_text(entry.get("region")),
"variant_group": _json_text(entry.get("variant_group")),
"selection_slot": _json_text(entry.get("selection_slot")),
"size": entry.get("size"),
"sha1": _json_text(entry.get("sha1")),
"sha256": _json_text(entry.get("sha256")),
@@ -1371,7 +1371,7 @@ def _write_sqlite_export(
CREATE TABLE platform_files (
platform_id TEXT NOT NULL, system TEXT NOT NULL, name TEXT NOT NULL,
destination TEXT NOT NULL, required INTEGER NOT NULL,
region TEXT, variant_group TEXT, size INTEGER,
region TEXT, selection_slot TEXT, size INTEGER,
sha1 TEXT, sha256 TEXT, md5 TEXT, crc32 TEXT,
status TEXT, category TEXT, collection_path TEXT, resolved_sha1 TEXT,
zipped_file TEXT, hash_match_mandatory INTEGER, mandatory_declared INTEGER,
@@ -1454,7 +1454,7 @@ def _write_sqlite_export(
connection.executemany(
"INSERT INTO platform_files VALUES "
"(:platform_id, :system, :name, :destination, :required, :region, "
":variant_group, :size, :sha1, :sha256, :md5, :crc32, :status, :category, "
":selection_slot, :size, :sha1, :sha256, :md5, :crc32, :status, :category, "
":collection_path, :resolved_sha1, :zipped_file, :hash_match_mandatory, "
":mandatory_declared, :curated, :shared_group, :mirror_of, :zip_digest)",
platform_files,
@@ -1698,7 +1698,7 @@ def generate_data_exports(
_csv_document(
[
"platform_id", "platform", "system", "name", "destination",
"required", "region", "variant_group", "size", "sha1",
"required", "region", "selection_slot", "size", "sha1",
"sha256", "md5", "crc32", "status", "category",
"collection_path", "resolved_sha1", *ENTRY_EXPORT_FIELDS,
],
+1 -1
View File
@@ -643,7 +643,7 @@ def _launcher_entry(
entry = {
key: report.get(key)
for key in (
"profile", "emulator", "systems", "system", "region", "variant_group",
"profile", "emulator", "systems", "system", "region", "selection_slot",
"required", "hle_fallback", "category",
)
}
+49 -10
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
from collections.abc import Iterable
from common import PROFILE_IDENTITY_FIELDS, _norm_system_id, entry_system_ids
from alternatives import core_file_spares, required_only_spares, set_fields
from common import archive_members, holds_required_members
from common import declared_root, expand_platform_declared_names, platform_declarations
from common import load_emulator_profiles
@@ -388,8 +389,12 @@ def hidden_extras(
)
if system_ids:
extras = extras_for_systems(extras, system_ids)
spares = core_file_spares(
extras,
lambda fe: extra_region_key(fe, region_prefix) in (region_drops or set()),
) if required_only else set()
for fe in extras:
if required_only and fe.get("required") is False:
if required_only and (fe.get("required") is False or id(fe) in spares):
continue
dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
if not dest or (region_drops and extra_region_key(fe, region_prefix) in region_drops):
@@ -640,7 +645,7 @@ def _agnostic_scan_extras(
"source_system": f.get("system"),
"source_systems": entry_system_ids(f, profile),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"selection_slot": f.get("selection_slot"),
"agnostic_scan": True,
}
extra["claim_order"] = (4, 0, emu_name)
@@ -796,7 +801,7 @@ def _companion_extras(
"source_system": seed.get("system"),
"source_systems": entry_system_ids(seed, profile),
"region": seed.get("region"),
"variant_group": seed.get("variant_group"),
"selection_slot": seed.get("selection_slot"),
"agnostic_scan": True,
}
extra["claim_order"] = (4, 0, emu_name)
@@ -970,7 +975,7 @@ def _archive_prefix_copies(
"source_system": archive_entry.get("system"),
"source_systems": entry_system_ids(archive_entry, profile),
"region": archive_entry.get("region"),
"variant_group": archive_entry.get("variant_group"),
"selection_slot": archive_entry.get("selection_slot"),
}
members = archive_members(archive_name, profile.get("files", []))
if members:
@@ -1354,7 +1359,8 @@ def _collect_emulator_extras(
"source_system": f.get("system"),
"source_systems": entry_system_ids(f, profile),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"selection_slot": f.get("selection_slot"),
**set_fields(f, profile),
}
# The same identity the first pass carries: without it the copy
# resolves on its name alone and a same-named file of another
@@ -1522,7 +1528,8 @@ def _report_extra(u: dict, name: str, dest: str) -> dict:
"source_system": u.get("system"),
"source_systems": u.get("systems", []),
"region": u.get("region"),
"variant_group": u.get("variant_group"),
"selection_slot": u.get("selection_slot"),
**{k: u[k] for k in ("required_groups", "set_declared") if k in u},
}
if u.get("members"):
extra["members"] = u["members"]
@@ -1795,9 +1802,9 @@ def extra_region_groups(
return (spellings or {}).get(_norm_system_id(sys_id), sys_id)
emulator = extra.get("source_emulator", "")
variant = extra.get("variant_group")
variant = extra.get("selection_slot")
if variant:
# A variant group is one slot of one core, whatever system each
# A selection slot is one slot of one core, whatever system each
# member is filed under: PicoDrive files its US Mega CD BIOS under
# sega-segacd and the EU/JP ones under sega-megacd, and find_bios
# picks one of the twelve. Keyed as emulator mode keys it.
@@ -1826,7 +1833,7 @@ def emulator_region_drops(
) -> set[str]:
"""Destinations an emulator-mode `--region` withdraws.
One group per system (or explicit variant_group) inside each profile, so
One group per system (or explicit selection_slot) inside each profile, so
a multi-system core such as O2EM keeps a fallback BIOS for every system
instead of letting one matching system empty another. The emulator pack
and `verify --emulator` both read this, keyed by pack destination.
@@ -1864,6 +1871,38 @@ def emulator_region_drops(
return region_mod.resolve_region_drops(region_groups, region_index, regions, held)
def emulator_required_spares(
emu_name: str,
profile: dict,
files: list[dict],
standalone: bool,
region_drops: set[str],
db: dict,
validation_index: dict,
zip_contents: dict | None,
data_registry: dict | None,
) -> set[int]:
"""The members of required sets an emulator --required-only pack leaves
out: one held member stays per set, the first the code loads among those
the pack still carries after the region filter. The pack and
verify --emulator both read this."""
from alternatives import groups_of, rank_in, required_groups
from packpaths import _resolve_destination
loose = [fe for fe in files if not fe.get("archive") and groups_of(fe)]
if not loose:
return set()
return required_only_spares(
loose,
owner=lambda _fe: emu_name,
held=lambda fe: _emulator_ships(
fe, emu_name, profile, db, validation_index, zip_contents, data_registry,
),
gone=lambda fe: _resolve_destination(fe, standalone) in region_drops,
sets=lambda fe: {g: rank_in(fe, g) for g in required_groups(fe, profile)},
)
def _emulator_ships(
file_entry: dict, emu_name: str, profile: dict, db: dict,
validation_index: dict, zip_contents: dict | None, data_registry: dict | None,
@@ -1904,7 +1943,7 @@ def _emulator_region_group(
Grouped together, geargrafx's LaserActive pac-n1.bin was dropped by
--emulator --region us and kept by --platform --region us.
"""
variant = file_entry.get("variant_group")
variant = file_entry.get("selection_slot")
if variant:
return f"{emu_name}:variant:{variant}"
system = file_entry.get("system")
+37 -6
View File
@@ -127,7 +127,7 @@ def resolve_slot_drops(
# A slot is a system AND a declared region: the Japanese and American
# PlayStation BIOS are not alternatives to each other, and comparing their
# ranks across regions would pick one territory's file for another's.
tiers: dict[tuple[str, tuple[str, ...], tuple], list[tuple[str, int | None]]] = {}
bucketed: dict[tuple[str, tuple[str, ...]], list[tuple[str, int | None, frozenset]]] = {}
for group_id, members in groups.items():
for destination, name in members:
if held is not None and destination not in held:
@@ -136,14 +136,20 @@ def resolve_slot_drops(
if entry is None:
keep.add(destination)
continue
# A rank orders the members of one set: px68k-sdl ranks its IPL
# and its font ROM lists each from 1, and the two never compete.
tier = tuple(sorted(entry["regions"]))
sets = tuple(sorted(entry.get("sets") or ()))
rank = None if entry.get("conflict") else entry["rank"]
tiers.setdefault((group_id, tier, sets), []).append((destination, rank))
bucketed.setdefault((group_id, tier), []).append(
(destination, rank, frozenset(entry.get("sets") or ()))
)
# A rank orders the members of one set: px68k-sdl ranks its IPL and its
# font ROM lists each from 1, and the two never compete. Candidates are
# compared when they share a set, and those in no set among themselves.
tiers: dict[tuple[str, tuple[str, ...], int], list[tuple[str, int | None]]] = {}
for (group_id, tier), found in bucketed.items():
for part, component in enumerate(_by_shared_set(found)):
tiers[(group_id, tier, part)] = component
for (group_id, tier, _sets), candidates in tiers.items():
for (group_id, tier, _part), candidates in tiers.items():
if len(candidates) < 2:
keep.update(dest for dest, _p in candidates)
continue
@@ -167,3 +173,28 @@ def resolve_slot_drops(
drop.update(dest for dest, _p in candidates if dest != winners[0])
return drop - keep, sorted(undecidable)
def _by_shared_set(
found: list[tuple[str, int | None, frozenset]],
) -> list[list[tuple[str, int | None]]]:
"""The candidates of one tier split where no set links them."""
parent = list(range(len(found)))
def root(i: int) -> int:
while parent[i] != i:
parent[i] = parent[parent[i]]
i = parent[i]
return i
first_of: dict[object, int] = {}
for i, (_dest, _rank, sets) in enumerate(found):
for key in sets or {None}:
if key in first_of:
parent[root(i)] = root(first_of[key])
else:
first_of[key] = i
parts: dict[int, list[tuple[str, int | None]]] = {}
for i, (dest, rank, _sets) in enumerate(found):
parts.setdefault(root(i), []).append((dest, rank))
return list(parts.values())
+19 -13
View File
@@ -124,27 +124,33 @@ def _systems_outside_profile(path: Path, document: object) -> list[str]:
def _group_requirement_split(path: Path, document: object) -> list[str]:
"""A group whose members disagree on required.
"""A set named by a member and not declared, declared and never used,
or a member that writes a required of its own.
The members of a group are one need, so required states the set's and
reads the same on each. A true on the first name only wrote the search
order into required, and the other names read as optional files.
The requirement belongs to the set, in the profile's groups map, and a
member's is derived from it. A value on the member is a second source
that can contradict the first: a true on the first name only wrote the
search order into required, and the other names read as optional.
"""
from alternatives import groups_of
if not isinstance(document, dict):
return []
seen: dict[str, set[bool]] = {}
for entry in document.get("files") or []:
label = path.relative_to(ROOT)
declared = set((document.get("groups") or {}).keys())
used: set[str] = set()
out: list[str] = []
for index, entry in enumerate(document.get("files") or []):
if not isinstance(entry, dict):
continue
for group in groups_of(entry):
seen.setdefault(group, set()).add(bool(entry.get("required", True)))
return [
f"{path.relative_to(ROOT)}: group {group} mixes required true and false"
for group, values in sorted(seen.items())
if len(values) > 1
]
names = groups_of(entry)
used.update(names)
if names and "required" in entry:
out.append(f"{label}: files[{index}] ({entry.get('name')}) is in a group "
"and writes required: the set's requirement is in groups")
out.extend(f"{label}: group {g} has no entry in groups" for g in sorted(used - declared))
out.extend(f"{label}: groups declares {g}, which no entry names" for g in sorted(declared - used))
return out
def _checks_without_values(path: Path, document: object) -> list[str]:
+42 -11
View File
@@ -37,7 +37,14 @@ import slots
sys.path.insert(0, os.path.dirname(__file__))
from hashing import parse_md5_list
from alternatives import set_keys, settle_sets, slot_set_keys
from alternatives import (
core_file_spares,
set_fields,
set_keys,
set_requirements,
settle_sets,
slot_set_keys,
)
from common import (
_foreign_tree,
_owner_tokens,
@@ -591,7 +598,7 @@ def find_undeclared_files(
f.get("archive"),
dest,
f.get("system"),
f.get("variant_group"),
f.get("selection_slot"),
region_key,
# Below the save directory each core is handed a folder
# of its own: two cores reading hiscore.dat there read
@@ -625,7 +632,7 @@ def find_undeclared_files(
archive,
f.get("system_dir"),
f.get("system"),
f.get("variant_group"),
f.get("selection_slot"),
region_key,
is_standalone,
)
@@ -640,7 +647,7 @@ def find_undeclared_files(
"systems": entry_system_ids(f, profile),
"system": f.get("system"),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"selection_slot": f.get("selection_slot"),
"name": archive,
"archive": archive,
"path": system_dir_path(f, archive),
@@ -738,7 +745,7 @@ def find_undeclared_files(
"systems": entry_system_ids(f, profile),
"system": f.get("system"),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"selection_slot": f.get("selection_slot"),
"name": fname,
"path": dest,
"required": f.get("required", False),
@@ -748,6 +755,7 @@ def find_undeclared_files(
"in_repo": in_repo,
"note": f.get("note", ""),
"checks": sorted(checks) if checks else [],
**set_fields(f, profile),
"source_ref": f.get("source_ref"),
"expected": _build_expected(f, checks),
**_identity_of(f),
@@ -785,7 +793,7 @@ def find_undeclared_files(
undeclared = [u for u in undeclared if id(u) not in settled]
undeclared.extend(added)
undeclared = _settle_core_sets(undeclared, set_records, settled)
undeclared = _settle_core_sets(undeclared, set_records, settled, profiles)
for entry in sorted(archive_entries.values(), key=lambda e: e["name"]):
undeclared.append(entry)
@@ -797,6 +805,7 @@ def _settle_core_sets(
undeclared: list[dict],
records: list[tuple[tuple, dict, bool, tuple[dict, dict | None] | None]],
launcher_settled: set[int],
profiles: dict[str, dict],
) -> list[dict]:
"""The core complement with each set of alternatives settled.
@@ -811,14 +820,23 @@ def _settle_core_sets(
excused, standing = settle_sets([
(keys, f, fills or (ref is not None and id(ref[0]) in launcher_settled))
for keys, f, fills, ref in grouped
])
], lambda key: set_requirements(profiles.get(key[0][0], {})).get(key[1], True))
declared_sets = {
key for keys, _f, _fills, ref in grouped if ref is None for key in keys
}
dropped: set[int] = set()
for position, (_keys, _f, _fills, ref) in enumerate(grouped):
for position, (keys, _f, _fills, ref) in enumerate(grouped):
if ref is None:
continue
entry, host = ref
required_keys = [k for k in keys if k[1] in entry.get("required_groups", ())]
if required_keys and all(k in declared_sets for k in required_keys):
# The platform declares a member of each of its required sets:
# under --required-only that declaration carries them.
entry["set_declared"] = True
if position in standing:
entry["alternatives"] = standing[position]
entry["alternatives"] = standing[position].others
entry["required"] = standing[position].required
if position not in excused:
continue
if host is not None:
@@ -1548,6 +1566,10 @@ def verify_platform(
)
not in region_drops
]
if required_only:
# One member of a required set, as the builder keeps it.
spares = core_file_spares(undeclared)
undeclared = [u for u in undeclared if id(u) not in spares]
_mark_unplaced(config, undeclared, profiles)
from packextras import ExtraResolver
@@ -2343,6 +2365,14 @@ def verify_emulator(
if fe.get("archive")
or _resolve_destination(fe, standalone) not in region_drops
]
if required_only:
from packextras import emulator_required_spares
spares = emulator_required_spares(
emu_name, profile, files, standalone, region_drops, db,
validation_index, zip_contents, data_registry,
)
files = [fe for fe in files if id(fe) not in spares]
# Check data directories (only notice if not cached)
for dd in deliverable(
@@ -2645,13 +2675,14 @@ def verify_emulator(
and result.get("reason") == fallback_reason),
)
for _dest, result, _required, _hle, entries in placed_results
])
], lambda key: set_requirements(profile).get(key[1], True))
for position, (dest, result, required, hle, _fe) in enumerate(placed_results):
if position in excused and result["status"] == Status.MISSING:
set_members.append(result)
continue
if position in standing:
result["alternatives"] = standing[position]
result["alternatives"] = standing[position].others
result["required"] = required = standing[position].required
details.append(result)
# Aggregate by destination (path if available, else name)