feat: install only what the installed cores read

This commit is contained in:
Abdessamad Derraz committed 2026-10-16 05:02:50 +02:00
1 parent 058ac673c4
commit fbe432ccd0
23 files changed
+16680 -2185

No files matched your search

+53 -54
View File
@@ -1518,15 +1518,7 @@ def resolve_platform_cores(
}
if target_cores is not None:
# Build reverse index: upstream name -> profile key
# Upstream sources (buildbot, es_systems) may use different names
# than our profile keys (e.g., mednafen_psx vs beetle_psx).
# The profiles' cores: field lists these alternate names.
index = upstream_profile_index(profiles, include_aliases=True)
expanded = {
profile for c in target_cores for profile in index.get(c, {c})
}
result = result & expanded
result = result & designated_profiles(profiles, target_cores)
return result
@@ -1617,40 +1609,39 @@ def _norm_system_id(sid: str) -> str:
return s.replace("-", "")
def filter_systems_by_target(
def designated_profiles(profiles: dict[str, dict], names: set[str]) -> set[str]:
"""The profiles a list of upstream core names designates.
The buildbot and es_systems name cores their own way (`mednafen_psx` for
beetle_psx); a profile's `cores:` lists those names. A name no profile
claims stands for itself.
"""
index = upstream_profile_index(profiles, include_aliases=True)
return {profile for c in names for profile in index.get(c, {c})}
def system_core_reach(
systems: dict[str, dict],
profiles: dict[str, dict],
target_cores: set[str] | None,
platform_cores: set[str] | None = None,
) -> dict[str, dict]:
"""Filter platform systems to only those reachable by target cores.
platform_cores: set[str] | None,
) -> dict[str, set[str] | None]:
"""For each platform system, the cores whose presence keeps it.
A system is reachable if at least one core that emulates it is available
on the target. Only considers cores relevant to the platform (from
platform_cores). Systems whose cores are all outside the platform's
scope are kept (no information to exclude them).
Returns the filtered systems dict (or all if no target).
A system is reachable if at least one core that emulates it is available.
Only cores relevant to the platform (platform_cores) can rule a system
out: a system whose cores are all outside the platform's scope maps to
None, kept whatever is available, there being no information to exclude
it.
"""
if target_cores is None:
return systems
# Build reverse index for target core name resolution
index = upstream_profile_index(profiles, include_aliases=True)
expanded_target = {
profile for c in target_cores for profile in index.get(c, {c})
}
_norm_sid = _norm_system_id
# Build normalized system -> cores from ALL profiles
# Normalized system -> cores, from ALL profiles
norm_system_cores: dict[str, set[str]] = {}
for name, p in profiles.items():
if p.get("type") == "alias":
continue
for sid in p.get("systems", []):
norm_key = _norm_sid(sid)
norm_system_cores.setdefault(norm_key, set()).add(name)
norm_system_cores.setdefault(_norm_sid(sid), set()).add(name)
# Platform-scoped mapping (for distinguishing "no info" from "known but off-target")
norm_plat_system_cores: dict[str, set[str]] = {}
@@ -1658,14 +1649,16 @@ def filter_systems_by_target(
for name in platform_cores:
p = profiles.get(name, {})
for sid in p.get("systems", []):
norm_key = _norm_sid(sid)
norm_plat_system_cores.setdefault(norm_key, set()).add(name)
norm_plat_system_cores.setdefault(_norm_sid(sid), set()).add(name)
# A platform can file a system under an id no profile uses: Batocera's
# enterprise-64-128 is ep128emu's enterprise-64 and enterprise-128, and
# only CLK, off most targets, carries the joined id. The destinations a
# system declares name the cores that read its files. A bare name proves
# nothing (bios.bin belongs to many systems), so only a path counts.
# The evidence only keeps: a path shared with an off-target core says
# nothing of the other files, and Batocera's bbc runs on MAME, which no
# profile ties to bbcb.zip, while CLK reads two of its ROMs.
dest_cores: dict[str, set[str]] = {}
for name, p in profiles.items():
if p.get("type") == "alias":
@@ -1681,9 +1674,12 @@ def filter_systems_by_target(
if "/" in dest:
dest_cores.setdefault(dest, set()).add(name)
filtered = {}
reach: dict[str, set[str] | None] = {}
for sys_id, sys_data in systems.items():
norm_key = _norm_sid(sys_id)
if not norm_plat_system_cores.get(norm_key):
reach[sys_id] = None
continue
file_cores = {
core
for fe in sys_data.get("files", [])
@@ -1692,26 +1688,29 @@ def filter_systems_by_target(
(),
)
}
all_cores = norm_system_cores.get(norm_key, set())
plat_cores_here = norm_plat_system_cores.get(norm_key, set())
reach[sys_id] = file_cores | norm_system_cores.get(norm_key, set())
return reach
if file_cores & expanded_target:
# An on-target core reads the system's files. The evidence only
# keeps: a path shared with an off-target core says nothing of the
# other files, and Batocera's bbc runs on MAME, which no profile
# ties to bbcb.zip, while CLK reads two of its ROMs.
filtered[sys_id] = sys_data
elif not all_cores and not plat_cores_here:
# No profile maps to this system -keep it
filtered[sys_id] = sys_data
elif all_cores & expanded_target:
# At least one core is on the target
filtered[sys_id] = sys_data
elif not plat_cores_here:
# Platform resolution didn't find cores for this system -keep it
filtered[sys_id] = sys_data
# else: known cores exist but none are on the target -exclude
return filtered
def filter_systems_by_target(
systems: dict[str, dict],
profiles: dict[str, dict],
target_cores: set[str] | None,
platform_cores: set[str] | None = None,
) -> dict[str, dict]:
"""Filter platform systems to only those reachable by target cores.
Returns the filtered systems dict (or all if no target).
"""
if target_cores is None:
return systems
available = designated_profiles(profiles, target_cores)
reach = system_core_reach(systems, profiles, platform_cores)
return {
sys_id: sys_data
for sys_id, sys_data in systems.items()
if reach[sys_id] is None or reach[sys_id] & available
}
def platform_declarations(
+84
View File
@@ -60,6 +60,8 @@ from common import (
resolve_local_file,
sanitize_pack_path,
select_emulator_profiles,
system_core_reach,
upstream_profile_index,
yaml_load,
)
import packresolve
@@ -3236,6 +3238,46 @@ def _manifest_core_entries(
pack_only_sizes.append(file_size)
continue
# Every other core that reads this destination, in the order the
# builder meets them: a pack built without the earlier cores carries
# the first remaining one's copy. A claim joins the copy just before
# it when the bytes agree, and starts an alternative otherwise; it
# never joins an earlier copy, which would let it win over a core
# the builder tries first. Only a core's first claim can win: its
# later ones are reached only when that one is.
claimed = set(entry["cores"])
for claim in fe.get("other_claims", []):
claim_emu = claim.get("source_profile") or claim.get("source_emulator", "")
if not claim_emu or claim_emu in claimed:
continue
claimed.add(claim_emu)
if required_only and claim.get("required") is False:
continue
claim_path, claim_status = resolve_file(
claim, db, bios_dir, zip_contents,
dest_hint=claim.get("destination", ""), offline=offline,
)
if claim_status in ("not_found", "external", "user_provided") or not claim_path:
continue
claim_path, _ = validated_choice(
claim, claim_path, db, validation_index or {}, bios_dir, None, dest, owners,
)
copy, _size = _manifest_entry(
claim_path, manifest_dest, [claim_emu], db, repo_root, {}
)
if copy is None:
continue
alternatives = entry.setdefault("alternatives", [])
last = alternatives[-1] if alternatives else entry
if copy["sha1"] == last["sha1"]:
if claim_emu not in last["cores"]:
last["cores"].append(claim_emu)
continue
del copy["dest"]
alternatives.append(copy)
if not entry.get("alternatives"):
entry.pop("alternatives", None)
manifest_files.append(entry)
omitted_by_destination.pop(full_dest, None)
total_size += file_size
@@ -3553,6 +3595,9 @@ def generate_manifest(
]
now = _build_timestamp(db)
core_names, system_cores = _manifest_core_index(
config, emu_profiles, manifest_files, omitted_by_destination.values()
)
result: dict = {
"manifest_version": 2,
@@ -3581,11 +3626,50 @@ def generate_manifest(
"omitted_files": sorted(
omitted_by_destination.values(), key=lambda entry: entry["dest"]
),
"core_names": core_names,
"system_cores": system_cores,
"files": manifest_files,
}
return result
def _manifest_core_index(
config: dict,
profiles: dict[str, dict],
files: list[dict],
omitted,
) -> tuple[dict[str, list[str]], dict[str, list[str]]]:
"""What the installer needs to narrow by a list of core names.
A target list, or the cores found installed, names cores the way the
buildbot does, and a file names the profile that reads it: `core_names`
gives each profile the other names it goes by. `system_cores` gives each
platform system the cores whose presence keeps it, as
`filter_systems_by_target` decides; a system absent from it is kept
whatever is available.
"""
reach = system_core_reach(
config.get("systems", {}), profiles, _platform_cores(config, profiles)
)
system_cores = {
sys_id: sorted(cores) for sys_id, cores in sorted(reach.items())
if cores is not None
}
named = {
c for entry in [*files, *omitted]
for copy in [entry, *entry.get("alternatives", [])]
for c in copy.get("cores") or []
}
named |= {c for cores in system_cores.values() for c in cores}
aliases: dict[str, set[str]] = {}
for name, claimants in upstream_profile_index(profiles, include_aliases=True).items():
for profile in claimants & named:
if name != profile:
aliases.setdefault(profile, set()).add(name)
core_names = {profile: sorted(names) for profile, names in sorted(aliases.items())}
return core_names, system_cores
# Post-generation pack verification + manifest + SHA256SUMS
+132 -80
View File
@@ -158,6 +158,7 @@ def _agnostic_scan_extras(
seen_dests: set,
extras_prefix: str,
standalone_set: set[str] | None = None,
taken: dict[str, dict] | None = None,
) -> list[dict]:
"""Every interchangeable candidate a filename-agnostic core accepts.
@@ -261,33 +262,34 @@ def _agnostic_scan_extras(
folder = seed.rsplit("/", 1)[0] if "/" in seed else ""
dest = f"{folder}/{scan_name}" if folder else scan_name
full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest
if full_dest in seen_dests:
if full_dest in seen_dests and (taken is None or full_dest not in taken):
continue
seen_dests.add(full_dest)
extras.append(
{
"name": scan_name,
"destination": dest,
# The scan already holds the entry it selected, so it
# names it by content. Emitting the filename alone
# sent the packing step back to a by-name lookup, and
# three PS2 resources came back from another
# emulator's copy of the same filename.
"sha1": sha1,
"required": False,
"hle_fallback": False,
"source_emulator": profile.get("emulator", emu_name),
"source_profile": emu_name,
"source_system": f.get("system"),
"source_systems": list(profile.get("systems", [])),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"agnostic_scan": True,
}
)
extra = {
"name": scan_name,
"destination": dest,
# The scan already holds the entry it selected, so it
# names it by content. Emitting the filename alone
# sent the packing step back to a by-name lookup, and
# three PS2 resources came back from another
# emulator's copy of the same filename.
"sha1": sha1,
"required": False,
"hle_fallback": False,
"source_emulator": profile.get("emulator", emu_name),
"source_profile": emu_name,
"source_system": f.get("system"),
"source_systems": list(profile.get("systems", [])),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"agnostic_scan": True,
}
extra["claim_order"] = (4, 0, emu_name)
if not _take(extra, full_dest, seen_dests, taken):
continue
extras.append(extra)
extras.extend(_companion_extras(
_companions(path, scan_name, _companion_extensions(profile, fname), files_db),
folder, extras_prefix, seen_dests, profile, emu_name, f,
folder, extras_prefix, seen_dests, profile, emu_name, f, taken,
))
return extras
@@ -300,16 +302,16 @@ def _companion_extras(
profile: dict,
emu_name: str,
seed: dict,
taken: dict[str, dict] | None = None,
) -> list[dict]:
"""Extras for the companions of one scanned image, placed beside it."""
extras: list[dict] = []
for companion_sha1, companion in companions:
companion_dest = f"{folder}/{companion['name']}" if folder else companion["name"]
companion_full = f"{extras_prefix}/{companion_dest}" if extras_prefix else companion_dest
if companion_full in seen_dests:
if companion_full in seen_dests and (taken is None or companion_full not in taken):
continue
seen_dests.add(companion_full)
extras.append({
extra = {
"name": companion["name"],
"destination": companion_dest,
"sha1": companion_sha1,
@@ -322,7 +324,10 @@ def _companion_extras(
"region": seed.get("region"),
"variant_group": seed.get("variant_group"),
"agnostic_scan": True,
})
}
extra["claim_order"] = (4, 0, emu_name)
if _take(extra, companion_full, seen_dests, taken):
extras.append(extra)
return extras
@@ -368,6 +373,7 @@ def _archive_prefix_extras(
seen_dests: set,
extras_prefix: str,
by_name: dict,
taken: dict[str, dict] | None = None,
) -> list[dict]:
"""A second copy of an archive under the subdirectory a core reads.
@@ -398,25 +404,25 @@ def _archive_prefix_extras(
continue
dest = f"{prefix}/{archive_name}"
full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest
if full_dest in seen_dests:
if full_dest in seen_dests and (taken is None or full_dest not in taken):
continue
if not by_name.get(archive_name):
continue
seen_dests.add(full_dest)
extras.append(
{
"name": archive_name,
"destination": dest,
"required": True,
"hle_fallback": False,
"source_emulator": profile.get("emulator", emu_name),
"source_profile": emu_name,
"source_system": archive_entry.get("system"),
"source_systems": list(profile.get("systems", [])),
"region": archive_entry.get("region"),
"variant_group": archive_entry.get("variant_group"),
}
)
extra = {
"name": archive_name,
"destination": dest,
"required": True,
"hle_fallback": False,
"source_emulator": profile.get("emulator", emu_name),
"source_profile": emu_name,
"source_system": archive_entry.get("system"),
"source_systems": list(profile.get("systems", [])),
"region": archive_entry.get("region"),
"variant_group": archive_entry.get("variant_group"),
}
extra["claim_order"] = (3, 0, emu_name)
if _take(extra, full_dest, seen_dests, taken):
extras.append(extra)
return extras
@@ -517,6 +523,8 @@ def _collect_emulator_extras(
)
extras = []
seen_dests: set[str] = set(seen)
# Destination -> the extra that took it, to record later claimants on.
taken: dict[str, dict] = {}
for u in undeclared:
if not u["in_repo"]:
continue
@@ -538,38 +546,18 @@ def _collect_emulator_extras(
claimants[(u.get("emulator", ""), u.get("name", ""), u.get("path") or "")] = (
sanitize_pack_path(dest)
)
if full_dest in seen_dests:
extra = _report_extra(u, name, dest)
claims = [_report_extra(c, name, dest) for c in u.get("other_claims", [])]
# Where the builder meets each claim: the report lists the profiles in
# order, archives after the loose files.
for item in (extra, *claims):
item["claim_order"] = (1, 1 if u.get("archive") else 0, item["source_profile"])
if not _take(extra, full_dest, seen_dests, taken):
for claim in claims:
_another_claim(taken.get(full_dest), claim)
continue
seen_dests.add(full_dest)
extra = {
"name": name,
"destination": dest,
"required": u.get("required", False),
"hle_fallback": u.get("hle_fallback", False),
"source_emulator": u.get("emulator", ""),
"source_profile": u.get("profile", ""),
# Identity of the report entry this extra came from. The name
# alone does not identify it: Dolphin declares three IPL.bin that
# differ only by path, and collapsing them onto one key withdraws
# the wrong regional variant from the report.
"source_name": u.get("name", ""),
"source_path": u.get("path") or "",
"source_system": u.get("system"),
"source_systems": u.get("systems", []),
"region": u.get("region"),
"variant_group": u.get("variant_group"),
}
# Keep every reproducible identity constraint carried by the source
# profile. Some profiles expose CRC32/SHA-256 only through their
# validation block; dropping those fields here would let a same-named
# but different payload enter a generated pack.
expected = u.get("expected") or {}
for identity in PROFILE_IDENTITY_FIELDS:
declared = u.get(identity)
if declared in (None, "", []):
declared = expected.get(identity)
if declared not in (None, "", []):
extra[identity] = declared
for claim in claims:
_another_claim(extra, claim)
extras.append(extra)
# Second pass: find alternative destinations for files already in the pack.
@@ -624,7 +612,7 @@ def _collect_emulator_extras(
if dest == fname:
continue # no alternative destination
full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest
if full_dest in seen_dests:
if full_dest in seen_dests and full_dest not in taken:
continue
# Check file exists in repo or data dirs
if not (
@@ -633,7 +621,6 @@ def _collect_emulator_extras(
or by_path_suffix.get(dest)
):
continue
seen_dests.add(full_dest)
extra = {
"name": fname,
"destination": dest,
@@ -652,24 +639,89 @@ def _collect_emulator_extras(
for identity in PROFILE_IDENTITY_FIELDS:
if f.get(identity) not in (None, "", []):
extra[identity] = f[identity]
extras.append(extra)
extra["claim_order"] = (2, 0, emu_name)
if _take(extra, full_dest, seen_dests, taken):
extras.append(extra)
extras.extend(
_archive_prefix_extras(
profiles, relevant, covered_names, seen_dests, extras_prefix,
by_name,
by_name, taken,
)
)
extras.extend(
_agnostic_scan_extras(
profiles, relevant, db, by_name, seen_dests, extras_prefix,
standalone_set,
standalone_set, taken,
)
)
return extras
def _report_extra(u: dict, name: str, dest: str) -> dict:
"""The extra a cross-reference entry asks for at its destination."""
extra = {
"name": name,
"destination": dest,
"required": u.get("required", False),
"hle_fallback": u.get("hle_fallback", False),
"source_emulator": u.get("emulator", ""),
"source_profile": u.get("profile", ""),
# Identity of the report entry this extra came from. The name
# alone does not identify it: Dolphin declares three IPL.bin that
# differ only by path, and collapsing them onto one key withdraws
# the wrong regional variant from the report.
"source_name": u.get("name", ""),
"source_path": u.get("path") or "",
"source_system": u.get("system"),
"source_systems": u.get("systems", []),
"region": u.get("region"),
"variant_group": u.get("variant_group"),
}
# Keep every reproducible identity constraint carried by the source
# profile. Some profiles expose CRC32/SHA-256 only through their
# validation block; dropping those fields here would let a same-named
# but different payload enter a generated pack.
expected = u.get("expected") or {}
for identity in PROFILE_IDENTITY_FIELDS:
declared = u.get(identity)
if declared in (None, "", []):
declared = expected.get(identity)
if declared not in (None, "", []):
extra[identity] = declared
return extra
def _take(
extra: dict, full_dest: str, seen_dests: set[str], taken: dict[str, dict] | None
) -> bool:
"""Whether an extra takes its destination; a taken one records the claim."""
if full_dest in seen_dests:
if taken is not None:
_another_claim(taken.get(full_dest), extra)
return False
seen_dests.add(full_dest)
if taken is not None:
taken[full_dest] = extra
return True
def _another_claim(extra: dict | None, claim: dict) -> None:
"""Record a core that reads a destination another core's extra took.
The pack ships one file per destination, so the first claimant's extra
stands; the others are kept on it so the manifest can name every core
the file serves. A pack built for a target without the first core packs
the next one's copy, and an installer narrowing by core must keep it.
"""
if extra is None or claim.get("source_profile") == extra.get("source_profile"):
return
claims = extra.setdefault("other_claims", [])
claims.append(claim)
claims.sort(key=lambda c: c["claim_order"])
def _extra_system_ids(extra: dict) -> list[str]:
"""Return the narrowest system ownership preserved on a core extra."""
explicit = extra.get("source_system")
+35 -24
View File
@@ -505,6 +505,10 @@ def find_undeclared_files(
standalone_set = set(str(c) for c in config.get("standalone_cores", []))
undeclared = []
seen_files: set[tuple] = set()
# Key -> the entry reported for it. A later profile reading the same
# requirement is kept on that entry rather than counted again: a pack
# built without the first profile still has to name the file.
reported: dict[tuple, dict] = {}
# Track archives: archive_name -> {in_repo, emulator, files: [...], ...}
archive_entries: dict[tuple, dict] = {}
@@ -549,7 +553,11 @@ def find_undeclared_files(
f.get("variant_group"),
region_key,
)
if not fname or seen_key in seen_files:
if not fname:
continue
first = reported.get(seen_key)
duplicate = seen_key in seen_files
if duplicate and (first is None or first["profile"] == emu_name):
continue
verdict = _candidate_verdict(
f, fname, is_standalone, include_all, declared_names, dest
@@ -644,29 +652,32 @@ def find_undeclared_files(
in_repo = False
checks = _parse_validation(f.get("validation"))
undeclared.append(
{
"profile": emu_name,
"emulator": profile.get("emulator", emu_name),
"systems": list(profile.get("systems", [])),
"system": f.get("system"),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"name": fname,
"path": dest,
"required": f.get("required", False),
"hle_fallback": f.get("hle_fallback", False),
"category": f.get("category", "bios"),
"in_repo": in_repo,
"note": f.get("note", ""),
"checks": sorted(checks) if checks else [],
"source_ref": f.get("source_ref"),
"expected": _build_expected(f, checks),
**_identity_of(f),
"sha1": f.get("sha1"),
"md5": f.get("md5"),
}
)
entry = {
"profile": emu_name,
"emulator": profile.get("emulator", emu_name),
"systems": list(profile.get("systems", [])),
"system": f.get("system"),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"name": fname,
"path": dest,
"required": f.get("required", False),
"hle_fallback": f.get("hle_fallback", False),
"category": f.get("category", "bios"),
"in_repo": in_repo,
"note": f.get("note", ""),
"checks": sorted(checks) if checks else [],
"source_ref": f.get("source_ref"),
"expected": _build_expected(f, checks),
**_identity_of(f),
"sha1": f.get("sha1"),
"md5": f.get("md5"),
}
if duplicate:
first.setdefault("other_claims", []).append(entry)
continue
reported[seen_key] = entry
undeclared.append(entry)
# Append grouped archive entries
for entry in sorted(archive_entries.values(), key=lambda e: e["name"]):