mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
chore: merge round 2 fixes (integrity)
This commit is contained in:
commit
052f837e69
20 files changed
+698
-136
No files matched your search
+15
-1
@@ -936,11 +936,21 @@ def _affinity_tokens(text: str) -> set[str]:
|
||||
|
||||
|
||||
def _owner_tokens(file_entry: dict) -> set[str]:
|
||||
"""Tokens naming the profile that asks for the file."""
|
||||
"""Tokens naming the profile that asks for the file.
|
||||
|
||||
A profile named after another with a suffix (cxbx-reloaded-2023,
|
||||
mame4droid-2024) is a release of that emulator, and the tree named by
|
||||
the shorter name is its own: cxbx-reloaded-2023 was served xemu's
|
||||
encrypted EEPROM because bios/Other/cxbx-reloaded/ read as foreign.
|
||||
Only whole leading words count, so dolphin-mmjr2 is not dolphin-mmjr.
|
||||
"""
|
||||
tokens: set[str] = set()
|
||||
for owner in (file_entry.get("source_profile"), file_entry.get("source_emulator")):
|
||||
if owner:
|
||||
tokens |= _affinity_tokens(str(owner).replace(" ", ""))
|
||||
profile = str(file_entry.get("source_profile") or "").casefold()
|
||||
words = [word for word in re.split(r"[^0-9a-z]+", profile) if word]
|
||||
tokens |= {"".join(words[:end]) for end in range(1, len(words))}
|
||||
return tokens
|
||||
|
||||
|
||||
@@ -1105,6 +1115,10 @@ def _holding_members(paths: list[str], members: list[dict]) -> list[str]:
|
||||
|
||||
scores = {path: held(path) for path in paths}
|
||||
best = max(scores.values(), default=(0, 0))
|
||||
if best == (0, 0):
|
||||
# Not one declared member in any copy: the name is all that matches,
|
||||
# and galgames.zip holding only an assembler listing is not the set.
|
||||
return []
|
||||
return [path for path in paths if scores[path] == best]
|
||||
|
||||
|
||||
|
||||
+33
-14
@@ -45,7 +45,13 @@ from common import (
|
||||
select_emulator_profiles,
|
||||
)
|
||||
from hashing import parse_hash_list
|
||||
from validation import case_folded_slots, declaration_settles, outside_gap_scope
|
||||
from validation import (
|
||||
case_folded_slots,
|
||||
declaration_settles,
|
||||
installed_with_emulator,
|
||||
outside_gap_scope,
|
||||
unread_on,
|
||||
)
|
||||
|
||||
yaml = require_yaml()
|
||||
|
||||
@@ -429,7 +435,11 @@ class FileConsumers:
|
||||
for name, config in configs.items():
|
||||
for system_id, system in config.get("systems", {}).items():
|
||||
for entry in system.get("files", []):
|
||||
sha1 = self.sha1_of(entry, entry.get("destination", ""))
|
||||
# No destination hint, as the builder, the manifest and
|
||||
# verify resolve a declaration: with one, RetroBat's 63
|
||||
# mame/hash lists credited another revision than the one
|
||||
# its manifest installs.
|
||||
sha1 = self.sha1_of(entry)
|
||||
if sha1:
|
||||
self.platforms.setdefault(sha1, set()).add(name)
|
||||
self.platform_systems.setdefault(sha1, {}).setdefault(
|
||||
@@ -606,9 +616,15 @@ def _cross_reference_profile(
|
||||
report: dict,
|
||||
index: dict,
|
||||
standalone_cores: set[str] | None = None,
|
||||
platform: str = "",
|
||||
) -> None:
|
||||
"""Compare one emulator profile against what the platforms declare.
|
||||
|
||||
*platform* is the platform key under --platform, empty for the whole
|
||||
tree. It decides, through the functions verify reads, which entries the
|
||||
platform's launcher never opens and which ones its emulator package
|
||||
already installs.
|
||||
|
||||
Split out of cross_reference, where it was a 200-line loop body that
|
||||
carried the whole function's branching: every file is resolved by name,
|
||||
by path, by hash and by archive membership before it can be called a
|
||||
@@ -667,7 +683,7 @@ def _cross_reference_profile(
|
||||
if len(layouts) > 1:
|
||||
# One file read at one path by both builds counts once.
|
||||
seen_key = (effective_path.casefold(), seen_key)
|
||||
if not fname or seen_key in seen_files:
|
||||
if not fname or seen_key in seen_files or unread_on(f, platform):
|
||||
continue
|
||||
|
||||
# Collect unsourceable files separately (documented, not a gap)
|
||||
@@ -759,7 +775,7 @@ def _cross_reference_profile(
|
||||
"name": fname,
|
||||
"path": effective_path,
|
||||
"required": f.get("required", False),
|
||||
"bundled": bool(f.get("bundled")),
|
||||
"bundled": installed_with_emulator(f, platform),
|
||||
"note": f.get("note", ""),
|
||||
"source_ref": f.get("source_ref", ""),
|
||||
"in_platform": False,
|
||||
@@ -798,6 +814,7 @@ def cross_reference(
|
||||
all_declared: dict[str, list[tuple[int | None, str]]] | None = None,
|
||||
standalone_cores: set[str] | None = None,
|
||||
folded_slots: frozenset[str] = frozenset(),
|
||||
platform: str = "",
|
||||
) -> dict:
|
||||
"""Compare emulator profiles against platform declarations.
|
||||
|
||||
@@ -839,7 +856,8 @@ def cross_reference(
|
||||
}
|
||||
for emu_name, profile in profiles.items():
|
||||
_cross_reference_profile(
|
||||
emu_name, profile, declared, report, index, standalone_cores
|
||||
emu_name, profile, declared, report, index, standalone_cores,
|
||||
platform,
|
||||
)
|
||||
|
||||
return report
|
||||
@@ -870,14 +888,14 @@ def print_report(
|
||||
if gaps == 0 and not show_clean:
|
||||
continue
|
||||
|
||||
# Counted once, for the heading and the total alike: a file bundled
|
||||
# with its emulator is not one to source.
|
||||
emulator_bundled = sum(
|
||||
1 for g in data["gap_details"]
|
||||
if g.get("bundled") and g.get("source") == "missing"
|
||||
)
|
||||
# Counted once, for the heading and the total alike: a file installed
|
||||
# with its emulator is neither one to source nor one the pack brings,
|
||||
# held or not (validation.deliverable leaves it out of every pack).
|
||||
bundled_gaps = [g for g in data["gap_details"] if g.get("bundled")]
|
||||
emulator_bundled = len(bundled_gaps)
|
||||
counts = {key: data.get(f"gap_{key}", 0) for key in totals}
|
||||
counts["missing"] -= emulator_bundled
|
||||
for g in bundled_gaps:
|
||||
counts[g.get("source", "missing")] -= 1
|
||||
counts["bundled"] = emulator_bundled
|
||||
parts = [f"{count} {key}" for key, count in counts.items() if count]
|
||||
status = ", ".join(parts) if parts else "OK"
|
||||
@@ -894,7 +912,7 @@ def print_report(
|
||||
for g in data["gap_details"]:
|
||||
req = "*" if g["required"] else " "
|
||||
src = g.get("source", "missing").upper()
|
||||
if g.get("bundled") and g.get("source") == "missing":
|
||||
if g.get("bundled"):
|
||||
src = "BUNDLED"
|
||||
# One line per file: a note spanning lines fell to column 0
|
||||
# between entries and read as an emulator heading.
|
||||
@@ -918,7 +936,7 @@ def print_report(
|
||||
f"large_file: {totals['large_file']})")
|
||||
print(f" {totals['missing']} missing (need to be sourced)")
|
||||
if bundled:
|
||||
print(f" {bundled} bundled with their emulator, not collected")
|
||||
print(f" {bundled} bundled with their emulator, not packed")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -1035,6 +1053,7 @@ def main() -> None:
|
||||
all_declared=declared_names(args.platforms_dir, selected, db),
|
||||
standalone_cores=standalone_cores,
|
||||
folded_slots=folded_declared_slots(args.platforms_dir, selected, db),
|
||||
platform=config.get("_key", args.platform) if args.platform else "",
|
||||
)
|
||||
|
||||
if args.json:
|
||||
|
||||
@@ -872,19 +872,16 @@ def generate_pack(
|
||||
total_files += 1
|
||||
continue
|
||||
|
||||
local_path, status = resolve_file(
|
||||
local_path, status = resolve_declaration(
|
||||
file_entry,
|
||||
db,
|
||||
bios_dir,
|
||||
zip_contents,
|
||||
data_dir_registry=data_registry,
|
||||
data_registry,
|
||||
slot_overrides.get(full_dest),
|
||||
offline=offline,
|
||||
)
|
||||
|
||||
local_path, status = slots.apply_override(
|
||||
slot_overrides.get(full_dest), local_path, status
|
||||
)
|
||||
|
||||
if status == "external":
|
||||
file_ext = os.path.splitext(file_entry["name"])[1] or ""
|
||||
with tempfile.NamedTemporaryFile(
|
||||
@@ -4235,14 +4232,15 @@ def generate_manifest(
|
||||
file_entry: dict, sys_id: str, dest: str, override: str | None
|
||||
) -> tuple[str | None, str]:
|
||||
"""The file served once the slot is settled, or None and why not."""
|
||||
local_path, status = slots.apply_override(override, *resolve_file(
|
||||
local_path, status = resolve_declaration(
|
||||
file_entry,
|
||||
db,
|
||||
bios_dir,
|
||||
zip_contents,
|
||||
data_dir_registry=data_registry,
|
||||
data_registry,
|
||||
override,
|
||||
offline=offline,
|
||||
))
|
||||
)
|
||||
if status == "not_found" and not reads_file_contents(verification_mode):
|
||||
found = agnostic_substitute(file_entry, sys_id, db, manifest_profiles)
|
||||
if found:
|
||||
@@ -4909,6 +4907,7 @@ from packresolve import ( # noqa: E402,F401
|
||||
parse_hash_input,
|
||||
parse_hash_file,
|
||||
lookup_hashes,
|
||||
resolve_declaration,
|
||||
resolve_file,
|
||||
download_external,
|
||||
)
|
||||
|
||||
+33
-10
@@ -215,7 +215,8 @@ def hidden_extras(
|
||||
placed: set[str] | None = None,
|
||||
report: list[dict] | None = None,
|
||||
resolver: ExtraResolver | None = None,
|
||||
placed_by: dict[str, tuple[str, str, str]] | None = None,
|
||||
placed_kind: dict[str, str] | None = None,
|
||||
placed_local: dict[str, str] | None = None,
|
||||
) -> dict[tuple[str, str, str], str]:
|
||||
"""Report entries the builder holds and still leaves out, with the reason.
|
||||
|
||||
@@ -231,7 +232,11 @@ def hidden_extras(
|
||||
file left out. ``shipped`` maps the full destination of each platform
|
||||
file the pack carries to its local copy, or to None for a slot it keeps
|
||||
without one (user_provided, external). ``placed``, when given,
|
||||
receives the full destination of each held extra the pack carries.
|
||||
receives the full destination of each held extra the pack carries, and
|
||||
``placed_kind`` its category, carried by the extra itself: the second
|
||||
pass, the archive-prefix pass and the agnostic scan copy files no gap
|
||||
report entry names. ``placed_local`` receives the local copy the builder
|
||||
writes at each of those destinations, for the pack integrity check.
|
||||
``resolver`` is the builder's lookup. An extra it cannot resolve is
|
||||
skipped and takes no path, as the builder skips it.
|
||||
"""
|
||||
@@ -288,11 +293,10 @@ def hidden_extras(
|
||||
lower.add(full.lower())
|
||||
if placed is not None and local:
|
||||
placed.add(full)
|
||||
if placed_by is not None and local:
|
||||
placed_by[full] = (
|
||||
fe.get("source_emulator", ""), fe.get("source_name", ""),
|
||||
fe.get("source_path", ""),
|
||||
)
|
||||
if placed_kind is not None and local:
|
||||
placed_kind[full] = fe.get("category", "bios")
|
||||
if placed_local is not None and local:
|
||||
placed_local[full] = local
|
||||
return hidden
|
||||
|
||||
|
||||
@@ -469,6 +473,9 @@ def _agnostic_scan_extras(
|
||||
"hle_fallback": False,
|
||||
"source_emulator": profile.get("emulator", emu_name),
|
||||
"source_profile": emu_name,
|
||||
"source_name": fname,
|
||||
"source_path": f.get("path") or "",
|
||||
"category": f.get("category", "bios"),
|
||||
"source_system": f.get("system"),
|
||||
"source_systems": list(profile.get("systems", [])),
|
||||
"region": f.get("region"),
|
||||
@@ -622,6 +629,9 @@ def _companion_extras(
|
||||
"hle_fallback": False,
|
||||
"source_emulator": profile.get("emulator", emu_name),
|
||||
"source_profile": emu_name,
|
||||
"source_name": seed.get("name", ""),
|
||||
"source_path": seed.get("path") or "",
|
||||
"category": seed.get("category", "bios"),
|
||||
"source_system": seed.get("system"),
|
||||
"source_systems": list(profile.get("systems", [])),
|
||||
"region": seed.get("region"),
|
||||
@@ -734,6 +744,10 @@ def _archive_prefix_extras(
|
||||
"hle_fallback": False,
|
||||
"source_emulator": profile.get("emulator", emu_name),
|
||||
"source_profile": emu_name,
|
||||
# Keyed as the gap report keys an archive: name and path alike.
|
||||
"source_name": archive_name,
|
||||
"source_path": archive_name,
|
||||
"category": archive_entry.get("category", "bios"),
|
||||
"source_system": archive_entry.get("system"),
|
||||
"source_systems": list(profile.get("systems", [])),
|
||||
"region": archive_entry.get("region"),
|
||||
@@ -1032,6 +1046,10 @@ def _collect_emulator_extras(
|
||||
"hle_fallback": f.get("hle_fallback", False),
|
||||
"source_emulator": profile.get("emulator", emu_name),
|
||||
"source_profile": emu_name,
|
||||
# The key the gap report gives the same entry.
|
||||
"source_name": fname,
|
||||
"source_path": raw,
|
||||
"category": f.get("category", "bios"),
|
||||
"source_system": f.get("system"),
|
||||
"source_systems": list(profile.get("systems", [])),
|
||||
"region": f.get("region"),
|
||||
@@ -1161,6 +1179,7 @@ def _report_extra(u: dict, name: str, dest: str) -> dict:
|
||||
# the wrong regional variant from the report.
|
||||
"source_name": u.get("name", ""),
|
||||
"source_path": u.get("path") or "",
|
||||
"category": u.get("category", "bios"),
|
||||
"source_system": u.get("system"),
|
||||
"source_systems": u.get("systems", []),
|
||||
"region": u.get("region"),
|
||||
@@ -1229,14 +1248,18 @@ def _kept(entries: list[dict], required_only: bool) -> list[dict]:
|
||||
return [e for e in entries if e.get("required") is not False]
|
||||
|
||||
|
||||
def _ships(config: dict, file_entry: dict, db: dict, dest: str) -> bool:
|
||||
def _ships(config: dict, file_entry: dict, db: dict) -> bool:
|
||||
"""Whether the builder packs this declaration: resolved, and not refused
|
||||
by a hash frontend. A dump whose md5 Recalbox would reject is left out
|
||||
of the pack, so it cannot win a regional slot over a file that ships.
|
||||
"""
|
||||
from packresolve import resolve_declaration
|
||||
from validation import baseline_verdict
|
||||
|
||||
path, status = resolve_local_file(file_entry, db, dest_hint=dest)
|
||||
# The builder's resolution: no destination hint, release assets included.
|
||||
path, status = resolve_declaration(
|
||||
file_entry, db, "bios", None, None, None, offline=True,
|
||||
)
|
||||
if not path:
|
||||
return False
|
||||
return baseline_verdict(
|
||||
@@ -1288,7 +1311,7 @@ def platform_region_groups(
|
||||
if dest:
|
||||
members.append((dest, file_entry.get("name", "")))
|
||||
if held_out is not None and db is not None and _ships(
|
||||
config, file_entry, db, dest
|
||||
config, file_entry, db
|
||||
):
|
||||
held_out.add(dest)
|
||||
|
||||
|
||||
@@ -345,6 +345,46 @@ def resolve_file(
|
||||
return None, "not_found"
|
||||
|
||||
|
||||
def apply_override(
|
||||
override: str | None, local_path: str | None, status: str
|
||||
) -> tuple[str | None, str]:
|
||||
"""The file a pack serves at a destination once its slot is settled.
|
||||
|
||||
An entry the platform stores outside the collection, or a release asset
|
||||
fetched for its declared hash, keeps its own answer.
|
||||
"""
|
||||
if override and status not in ("external", "release_asset"):
|
||||
return override, "slot_arbitrated"
|
||||
return local_path, status
|
||||
|
||||
|
||||
def resolve_declaration(
|
||||
file_entry: dict,
|
||||
db: dict,
|
||||
bios_dir: str,
|
||||
zip_contents: dict | None,
|
||||
data_dir_registry: dict | None,
|
||||
override: str | None,
|
||||
*,
|
||||
offline: bool | None = None,
|
||||
) -> tuple[str | None, str]:
|
||||
"""The file a platform pack serves for one of the platform's declarations.
|
||||
|
||||
Read by the builder, the install manifest, verify and the region pass.
|
||||
verify resolved with common.resolve_local_file and put the slot override
|
||||
first: without the release-asset tier a gitignored file absent from bios/
|
||||
read as missing there while the pack shipped the cached asset.
|
||||
"""
|
||||
return apply_override(override, *resolve_file(
|
||||
file_entry,
|
||||
db,
|
||||
bios_dir,
|
||||
zip_contents,
|
||||
data_dir_registry=data_dir_registry,
|
||||
offline=offline,
|
||||
))
|
||||
|
||||
|
||||
def preferred_entries(
|
||||
pack_systems: dict,
|
||||
db: dict,
|
||||
|
||||
+96
-57
@@ -10,10 +10,7 @@ from collections import Counter
|
||||
|
||||
from auto_fetch import DEFAULT_BIOS_DIR
|
||||
from artifacts import _build_timestamp
|
||||
from packextras import _collect_emulator_extras
|
||||
from packextras import _detect_extras_prefix
|
||||
from packpaths import _has_path_conflict
|
||||
from packpaths import _register_path
|
||||
from ziptools import build_zip_contents_index
|
||||
from ziptools import check_inside_zip
|
||||
from nativemode import digest_algorithm
|
||||
@@ -525,6 +522,81 @@ def _structural_errors(zf: zipfile.ZipFile, zip_set: set) -> list[str]:
|
||||
return errors
|
||||
|
||||
|
||||
def _builder_extras(
|
||||
config: dict,
|
||||
emulators_dir: str,
|
||||
db: dict,
|
||||
emu_profiles: dict,
|
||||
baseline_groups: dict[str, list[dict]],
|
||||
base_dest: str,
|
||||
is_flat: bool,
|
||||
zip_set: set[str],
|
||||
zip_lower: dict[str, str],
|
||||
target_cores: set[str] | None,
|
||||
region_drops: set[str],
|
||||
data_registry: dict | None,
|
||||
) -> dict[str, str]:
|
||||
"""{full destination: local copy} of each core extra the builder writes.
|
||||
|
||||
The platform's paths come first, as in the builder: a destination the
|
||||
pack holds whose declarations resolve to a file, or that the platform
|
||||
keeps for a user_provided or external entry. One whose declarations
|
||||
resolve to nothing is left free, since the builder then places a core's
|
||||
file there.
|
||||
"""
|
||||
from packextras import ExtraResolver, hidden_extras
|
||||
from packresolve import resolve_declaration
|
||||
from validation import _build_validation_index, destination_owners
|
||||
|
||||
shipped: dict[str, str | None] = {}
|
||||
for expected, declarations in baseline_groups.items():
|
||||
if expected not in zip_set and expected.lower() not in zip_lower:
|
||||
continue
|
||||
unflat = (
|
||||
f"{base_dest}/{expected}"
|
||||
if is_flat and base_dest and not expected.startswith(f"{base_dest}/")
|
||||
else expected
|
||||
)
|
||||
if any(fe.get("storage") in ("user_provided", "external") for fe in declarations):
|
||||
shipped[unflat] = None
|
||||
continue
|
||||
local = next(
|
||||
(
|
||||
found for fe in declarations
|
||||
if (found := resolve_declaration(
|
||||
fe, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True,
|
||||
)[0])
|
||||
),
|
||||
None,
|
||||
)
|
||||
if local:
|
||||
shipped[unflat] = local
|
||||
platform_profiles = {
|
||||
name: emu_profiles[name] for name in resolve_platform_cores(config, emu_profiles)
|
||||
}
|
||||
placed_local: dict[str, str] = {}
|
||||
hidden_extras(
|
||||
config, emulators_dir, db, emu_profiles, shipped, target_cores,
|
||||
region_drops or None,
|
||||
resolver=ExtraResolver(
|
||||
db, DEFAULT_BIOS_DIR, build_zip_contents_index(db), data_registry,
|
||||
_build_validation_index(platform_profiles),
|
||||
destination_owners(platform_profiles), True,
|
||||
),
|
||||
placed_local=placed_local,
|
||||
)
|
||||
return placed_local
|
||||
|
||||
|
||||
def _same_content(zf: zipfile.ZipFile, member: str, local: str) -> bool:
|
||||
"""Whether a pack member carries the bytes of a local copy, or the
|
||||
members of an archive the builder normalized."""
|
||||
with open(local, "rb") as handle:
|
||||
if zf.read(member) == handle.read():
|
||||
return True
|
||||
return local.lower().endswith(".zip") and _same_members(zf, member, local)
|
||||
|
||||
|
||||
def verify_pack_against_platform(
|
||||
zip_path: str,
|
||||
platform_name: str,
|
||||
@@ -707,66 +779,33 @@ def verify_pack_against_platform(
|
||||
):
|
||||
errors.append(member_errors[0])
|
||||
|
||||
# 2. Core extras presence (files from emulator profiles, in repo)
|
||||
# Mirror the pack builder's skip logic: only count files that
|
||||
# can actually be resolved and don't have path conflicts.
|
||||
# 2. Core extras: the builder's own placement replayed
|
||||
# (packextras.hidden_extras, which verify reads too), after the
|
||||
# platform files it wrote. Each placed extra must sit at its path
|
||||
# with the bytes the builder resolved: an empty seen set and a
|
||||
# presence test credited a core file at a path where the pack
|
||||
# holds the platform's other dump.
|
||||
core_checked = 0
|
||||
core_present = 0
|
||||
if db is not None:
|
||||
core_files = _collect_emulator_extras(
|
||||
config,
|
||||
emulators_dir,
|
||||
db,
|
||||
set(),
|
||||
base_dest,
|
||||
emu_profiles,
|
||||
target_cores=target_cores,
|
||||
placed_local = _builder_extras(
|
||||
config, emulators_dir, db, emu_profiles, baseline_groups,
|
||||
base_dest, is_flat, zip_set, zip_lower, target_cores,
|
||||
region_drops, data_registry,
|
||||
)
|
||||
seen_conformance: set[str] = set(zip_set)
|
||||
seen_parents: set[str] = set()
|
||||
for n in zip_set:
|
||||
parts = n.split("/")
|
||||
for i in range(1, len(parts)):
|
||||
seen_parents.add("/".join(parts[:i]))
|
||||
extras_pfx = _detect_extras_prefix(config, base_dest)
|
||||
for fe in core_files:
|
||||
raw_dest = fe.get("destination", fe.get("name", ""))
|
||||
dest = sanitize_pack_path(raw_dest)
|
||||
if not dest:
|
||||
continue
|
||||
if region_drops and sanitize_pack_path(dest) in region_drops:
|
||||
continue
|
||||
# The builder's own placement: the prefix, then the flattening
|
||||
# of a pack written at the BIOS root. A destination that
|
||||
# already starts with the prefix (aethersx2's bios/<image>)
|
||||
# lands at the root of a flat pack, not under bios/.
|
||||
if extras_pfx and not dest.startswith(f"{extras_pfx}/"):
|
||||
full = f"{extras_pfx}/{dest}"
|
||||
else:
|
||||
full = dest
|
||||
full = _flat(full, base_dest, is_flat)
|
||||
# Skip path conflicts (same logic as pack builder)
|
||||
if _has_path_conflict(full, seen_conformance, seen_parents):
|
||||
continue
|
||||
# Skip unresolvable files (game_data dirs, etc.)
|
||||
local_path, status = resolve_file(
|
||||
fe,
|
||||
db,
|
||||
"bios",
|
||||
{},
|
||||
dest_hint=raw_dest,
|
||||
data_dir_registry=data_registry,
|
||||
offline=True,
|
||||
)
|
||||
if status in ("not_found", "external", "user_provided"):
|
||||
continue
|
||||
for full, local in sorted(placed_local.items()):
|
||||
core_checked += 1
|
||||
if full in zip_set or full.lower() in zip_lower:
|
||||
core_present += 1
|
||||
seen_conformance.add(full)
|
||||
_register_path(full, seen_conformance, seen_parents)
|
||||
else:
|
||||
member = _flat(full, base_dest, is_flat)
|
||||
member = member if member in zip_set else zip_lower.get(member.lower())
|
||||
if member is None:
|
||||
errors.append(f"core missing: {full}")
|
||||
elif not _same_content(zf, member, local):
|
||||
errors.append(
|
||||
f"core file differs: {member} is not the {os.path.basename(local)}"
|
||||
" the builder places there"
|
||||
)
|
||||
else:
|
||||
core_present += 1
|
||||
|
||||
checked = baseline_checked + core_checked
|
||||
present = baseline_present + core_present
|
||||
|
||||
@@ -578,20 +578,6 @@ def slot_choices(
|
||||
return choices
|
||||
|
||||
|
||||
def apply_override(
|
||||
override: str | None, local_path: str | None, status: str
|
||||
) -> tuple[str | None, str]:
|
||||
"""The file a pack serves at a destination once its slot is settled.
|
||||
|
||||
An entry the platform stores outside the collection, or a release asset
|
||||
fetched for its declared hash, keeps its own answer. The builder and the
|
||||
install manifest both read this.
|
||||
"""
|
||||
if override and status not in ("external", "release_asset"):
|
||||
return override, "slot_arbitrated"
|
||||
return local_path, status
|
||||
|
||||
|
||||
def _strongest(claims: list[Claim]) -> Claim:
|
||||
"""The profile claim with the strongest proof, the first emulator breaking ties.
|
||||
|
||||
|
||||
+33
-24
@@ -896,7 +896,7 @@ def _mark_hidden(
|
||||
region_drops: set[str] | None = None,
|
||||
report: list[dict] | None = None,
|
||||
resolver: object | None = None,
|
||||
placed_by: dict[str, tuple[str, str, str]] | None = None,
|
||||
placed_kind: dict[str, str] | None = None,
|
||||
) -> set[str]:
|
||||
"""What the builder holds but cannot place beside the files it packed.
|
||||
|
||||
@@ -911,7 +911,7 @@ def _mark_hidden(
|
||||
config, emulators_dir, db,
|
||||
emu_profiles if emu_profiles is not None
|
||||
else load_emulator_profiles(emulators_dir),
|
||||
shipped, target_cores, region_drops, placed, report, resolver, placed_by,
|
||||
shipped, target_cores, region_drops, placed, report, resolver, placed_kind,
|
||||
)
|
||||
for u in undeclared:
|
||||
reason = hidden.get(
|
||||
@@ -1058,18 +1058,20 @@ def _resolve_for_platform(
|
||||
) -> tuple[str | None, str]:
|
||||
"""Resolve one platform entry the way the pack builder does.
|
||||
|
||||
A slot the arbitration decided wins over the resolver, and a frontend that
|
||||
The builder's own resolution (packresolve.resolve_declaration), read from
|
||||
disk and the large-file cache: verify never downloads. A frontend that
|
||||
never reads the bytes takes the filename-free core's own image.
|
||||
"""
|
||||
from packresolve import resolve_declaration
|
||||
|
||||
override = slot_overrides.get(
|
||||
sanitize_pack_path(file_entry.get("destination", file_entry.get("name", "")))
|
||||
)
|
||||
if override:
|
||||
return override, "slot_arbitrated"
|
||||
local_path, status = resolve_local_file(
|
||||
file_entry, db, zip_contents, data_dir_registry=data_dir_registry
|
||||
local_path, status = resolve_declaration(
|
||||
file_entry, db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry,
|
||||
override, offline=True,
|
||||
)
|
||||
if local_path is None and not reads_file_contents(mode):
|
||||
if status == "not_found" and not reads_file_contents(mode):
|
||||
found = agnostic_substitute(file_entry, sys_id, db, platform_profiles)
|
||||
if found:
|
||||
return found[0], "agnostic_fallback"
|
||||
@@ -1342,7 +1344,7 @@ def verify_platform(
|
||||
_mark_unplaced(config, undeclared, profiles)
|
||||
from packextras import ExtraResolver
|
||||
|
||||
placed_by: dict[str, tuple[str, str, str]] = {}
|
||||
placed_kind: dict[str, str] = {}
|
||||
placed = _mark_hidden(
|
||||
config, undeclared, emulators_dir, db, emu_profiles, occupied, target_cores,
|
||||
region_drops, report_for_builder,
|
||||
@@ -1350,20 +1352,15 @@ def verify_platform(
|
||||
db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index,
|
||||
validation_owners,
|
||||
),
|
||||
placed_by,
|
||||
placed_kind,
|
||||
)
|
||||
# What the builder places from the cores, one per pack path, by category:
|
||||
# several entries claim one path (rom.key from three Amiga cores) and the
|
||||
# builder ships one file there.
|
||||
categories = {
|
||||
(u.get("emulator", ""), u.get("name", ""), u.get("path") or ""):
|
||||
u.get("category", "bios")
|
||||
for u in report_for_builder
|
||||
}
|
||||
# builder ships one file there. The category travels on the extra: a copy
|
||||
# the second pass makes has no gap report entry to look it up by.
|
||||
packed_extras = {"bios": 0, "game_data": 0}
|
||||
for key in placed_by.values():
|
||||
kind = "game_data" if categories.get(key) == "game_data" else "bios"
|
||||
packed_extras[kind] += 1
|
||||
for category in placed_kind.values():
|
||||
packed_extras["game_data" if category == "game_data" else "bios"] += 1
|
||||
if not reads_file_contents(mode):
|
||||
# An existence frontend is satisfied by any file at the path, and the
|
||||
# builder fills a free platform destination with a core's file.
|
||||
@@ -1825,6 +1822,7 @@ def verify_emulator(
|
||||
`system_ids` keeps the entries of those systems, as the system pack does.
|
||||
"""
|
||||
from packextras import expand_agnostic_entries
|
||||
from packresolve import resolve_file
|
||||
from packpaths import (
|
||||
_archive_destination,
|
||||
_register_path,
|
||||
@@ -1855,6 +1853,10 @@ def verify_emulator(
|
||||
file_severity: dict[str, str] = {}
|
||||
dest_to_name: dict[str, str] = {}
|
||||
data_dir_notices: list[str] = []
|
||||
# The paths the emulator pack writes a profile file to. A copy whose hash
|
||||
# the profile contradicts is written too (UNTESTED here, a DISCREPANCY in
|
||||
# the pack), so the status alone does not say what the pack holds.
|
||||
packed_dests: set[str] = set()
|
||||
|
||||
# The emulator pack withdraws these, and the report withdraws the same.
|
||||
region_drops: set[str] = set()
|
||||
@@ -1945,11 +1947,13 @@ def verify_emulator(
|
||||
archive = file_entry.get("archive")
|
||||
if archive and archive not in seen_archives:
|
||||
seen_archives.add(archive)
|
||||
local_path, resolve_status = resolve_local_file(
|
||||
local_path, resolve_status = resolve_file(
|
||||
archive_entry(archive, files),
|
||||
db,
|
||||
DEFAULT_BIOS_DIR,
|
||||
zip_contents,
|
||||
data_dir_registry=data_registry,
|
||||
offline=True,
|
||||
)
|
||||
required = any(
|
||||
f.get("archive") == archive and f.get("required", True)
|
||||
@@ -2006,6 +2010,7 @@ def verify_emulator(
|
||||
)
|
||||
elif local_path:
|
||||
occupy(dest, local_path, emu_name)
|
||||
packed_dests.add(dest)
|
||||
dest_to_name[dest] = archive
|
||||
cur = result["status"]
|
||||
prev = file_status.get(dest)
|
||||
@@ -2032,13 +2037,18 @@ def verify_emulator(
|
||||
continue
|
||||
|
||||
dest_hint = file_entry.get("path", "")
|
||||
local_path, resolve_status = resolve_local_file(
|
||||
# The emulator pack's resolution, release assets included.
|
||||
local_path, resolve_status = resolve_file(
|
||||
{**with_archive_members(file_entry, files), "source_profile": emu_name},
|
||||
db,
|
||||
DEFAULT_BIOS_DIR,
|
||||
zip_contents,
|
||||
dest_hint=dest_hint,
|
||||
data_dir_registry=data_registry,
|
||||
offline=True,
|
||||
)
|
||||
if resolve_status in ("not_found", "user_provided", "external"):
|
||||
local_path = None
|
||||
name = file_entry.get("name", "")
|
||||
required = file_entry.get("required", True)
|
||||
hle = file_entry.get("hle_fallback", False)
|
||||
@@ -2168,6 +2178,7 @@ def verify_emulator(
|
||||
block_reason = f"the pack holds {owner}'s {other_dest} at this path"
|
||||
elif not block and packed is not None:
|
||||
occupy(dest, packed, emu_name)
|
||||
packed_dests.add(dest)
|
||||
for result, required, hle in alternatives:
|
||||
if block_reason and result["status"] != Status.MISSING:
|
||||
result["status"] = Status.MISSING
|
||||
@@ -2235,9 +2246,7 @@ def verify_emulator(
|
||||
"total_files": total,
|
||||
# The paths the emulator pack fills with the profile's files, for the
|
||||
# pipeline's comparison with the pack built in memory.
|
||||
"pack_destinations": sorted(
|
||||
dest for dest, status in file_status.items() if status == Status.OK
|
||||
),
|
||||
"pack_destinations": sorted(packed_dests),
|
||||
"severity_counts": counts,
|
||||
"status_counts": status_counts,
|
||||
"details": details,
|
||||
|
||||
Reference in new issue
Block a user