kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-26 13:47:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge audit fixes (integrity)

Browse Source 
# Conflicts:
#	install/batocera.json
#	install/emudeck.json
#	install/lakka.json
#	install/recalbox.json
#	install/retroarch.json
#	install/retrobat.json
#	install/retrodeck.json
#	install/retropie.json
#	install/rocknix.json
#	install/romm.json
#	scripts/generate_pack.py
#	wiki/tools.md
This commit is contained in:
Abdessamad Derraz committed 2026-10-21 06:48:42 +02:00
2 parents 85a44d9bf7 88f9c40a78
commit 378a9e4da5
19 files changed
+1199 -101
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/cross_reference.py
+88 -22
View File
@@ -31,6 +31,7 @@ from common import (
get_mame_clone_map,
list_emulator_profiles,
list_registered_platforms,
load_data_dir_registry,
load_database,
load_emulator_profiles,
load_platform_config,
@@ -44,7 +45,7 @@ from common import (
select_emulator_profiles,
)
from hashing import parse_hash_list
from validation import declaration_settles, outside_gap_scope
from validation import case_folded_slots, declaration_settles, outside_gap_scope
yaml = require_yaml()
@@ -105,6 +106,25 @@ def declared_names(
)
def folded_declared_slots(
platforms_dir: str, platforms: list[str] | None, db: dict
) -> frozenset[str]:
"""The slots platforms whose filesystem folds case fill, any spelling.
RetroBat's 32X_G_BIOS.BIN is the file BlastEm opens as 32X_G_BIOS.bin,
and the exact-case name test called it undeclared.
"""
from common import platform_declarations
slots: set[str] = set()
for platform_name in platforms or list_registered_platforms(
platforms_dir, include_archived=True
):
config = load_platform_config(platform_name, platforms_dir)
slots |= case_folded_slots(config, platform_declarations(config, db))
return frozenset(slots)
def _build_supplemental_index(
data_root: str = "data", bios_root: str = "bios"
) -> set[str]:
@@ -178,21 +198,28 @@ def _resolve_source(
or ``None`` (not available anywhere).
A name hit whose size contradicts a size the emulator verifies is not the
file, so it does not count as available.
file, so it does not count as available. Nor is one whose bytes carry
none of the hashes the entry declares: a name never wins over a declared
hash, and zquestclassic's ending.mid read as held because Hydra Castle
Labyrinth has another file of that name.
"""
declared = _declared_content(file_entry or {})
def _name_hit(name: str) -> bool:
"""Whether the files indexed under *name* can be this entry."""
def _can_be(sha1s: list | str) -> bool:
if not file_entry or db_files is None:
return True
sha1s = by_name.get(name) or []
if not isinstance(sha1s, list):
sha1s = [sha1s]
return any(
name_match_size_ok(file_entry, db_files.get(sha1, {}).get("size"))
and _record_carries(sha1, db_files.get(sha1, {}), declared)
for sha1 in sha1s
)
def _name_hit(name: str) -> bool:
"""Whether the files indexed under *name* can be this entry."""
return _can_be(by_name.get(name) or [])
# bios/ via database.json by_name
if fname in by_name and _name_hit(fname):
return "bios"
@@ -209,7 +236,7 @@ def _resolve_source(
):
return "bios"
# bios/ via by_path_suffix (regional variants)
if by_path_suffix and fname in by_path_suffix:
if by_path_suffix and fname in by_path_suffix and _can_be(by_path_suffix[fname]):
return "bios"
# bios/ under the canonical MAME set name, as resolve_local_file does:
# a renamed archive is held once, under the name the dedup kept.
@@ -252,6 +279,23 @@ def _data_hit(
_HASH_FIELDS = ("sha1", "md5", "sha256", "crc32")
def _record_carries(sha1: str, record: dict, declared: dict[str, set[str]]) -> bool:
"""Whether a database record has one of the declared hashes, or none is declared."""
if not declared:
return True
actual = {
"sha1": sha1.lower(),
**{field: str(record.get(field) or "").lower() for field in ("md5", "sha256", "crc32")},
}
return any(
actual[field] and (
actual[field] in values
or (field == "md5" and any(len(v) < 32 and actual[field].startswith(v) for v in values))
)
for field, values in declared.items()
)
def _declared_content(file_entry: dict) -> dict[str, set[str]]:
"""The hashes an entry declares, lowercased, by algorithm."""
declared: dict[str, set[str]] = {}
@@ -326,9 +370,21 @@ class FileConsumers:
file by the evidence the pack uses, so a homonym is nobody's file.
"""
def __init__(self, db: dict, zip_contents: dict | None = None) -> None:
def __init__(
self, db: dict, zip_contents: dict | None = None,
data_dir_registry: dict | None = None,
) -> None:
self._db = db
self._zip_contents = zip_contents
# The builder resolves with the data trees: Dolphin's codehandler.bin
# comes from data/dolphin-sys, and without them the name step gave it
# Dolphin-MMJR's copy, which the site then said Dolphin reads.
self._data_dir_registry = (
data_dir_registry if data_dir_registry is not None
else load_data_dir_registry(
str(Path(__file__).resolve().parent.parent / DEFAULT_PLATFORMS_DIR)
)
)
self._sha1_by_path = {
path: sha1
for sha1, entry in db.get("files", {}).items()
@@ -362,7 +418,8 @@ class FileConsumers:
if owner:
entry = {**entry, "source_profile": owner}
path, status = resolve_local_file(
entry, self._db, self._zip_contents, dest_hint=dest
entry, self._db, self._zip_contents, dest_hint=dest,
data_dir_registry=self._data_dir_registry,
)
if not path or status in ("not_found", "hash_mismatch"):
return None
@@ -470,6 +527,7 @@ def entry_source(f: dict, index: dict) -> str | None:
{"indexes": {"by_path_suffix": by_path_suffix}}, path_field or fname
):
return "bios"
found = None
for candidate in [fname, *([path_field] if path_field != fname else []),
*(f.get("aliases") or [])]:
if not candidate:
@@ -478,12 +536,15 @@ def entry_source(f: dict, index: dict) -> str | None:
candidate, by_name, by_name_lower, data_names, by_path_suffix, f, db_files,
index.get("bios_dir"),
)
if source is not None:
if source == "bios":
return source
found = found or source
def values(field: str) -> list[str]:
return parse_hash_list(f.get(field))
# The declared content anywhere in bios/ comes before a data/ copy, as in
# the resolver: the hash steps run before the data directories.
held = (
any(index["by_md5"].get(v) for v in parse_md5_list(f.get("md5")))
or any(v in db_files for v in values("sha1"))
@@ -491,7 +552,7 @@ def entry_source(f: dict, index: dict) -> str | None:
or any(v in index["by_sha256"] for v in values("sha256"))
or any(index["by_crc32"].get(v) for v in values("crc32"))
)
return "bios" if held else None
return "bios" if held else found
def _resolve_archive_source(
@@ -629,7 +690,9 @@ def _cross_reference_profile(
# Check platform declaration: by name, by an alias the core answers
# to (RetroArch lists amiberry's cd32.rom as kick40060.CD32), or by
# archive.
in_platform = declaration_settles(f, fname, platform_names, effective_path)
in_platform = declaration_settles(
f, fname, platform_names, effective_path, index.get("folded_slots", frozenset())
)
if in_platform:
seen_files.add(seen_key)
@@ -736,6 +799,7 @@ def cross_reference(
data_names: set[str] | None = None,
all_declared: dict[str, list[tuple[int | None, str]]] | None = None,
standalone_cores: set[str] | None = None,
folded_slots: frozenset[str] = frozenset(),
) -> dict:
"""Compare emulator profiles against platform declarations.
@@ -772,6 +836,7 @@ def cross_reference(
"db_files": db_files,
"data_names": data_names,
"all_declared": all_declared,
"folded_slots": folded_slots,
"bios_dir": db.get("bios_dir"),
}
for emu_name, profile in profiles.items():
@@ -807,11 +872,16 @@ def print_report(
if gaps == 0 and not show_clean:
continue
parts = []
for key in ("bios", "data", "large_file", "missing"):
count = data.get(f"gap_{key}", 0)
if count:
parts.append(f"{count} {key}")
# Counted once, for the heading and the total alike: a file bundled
# with its emulator is not one to source.
emulator_bundled = sum(
1 for g in data["gap_details"]
if g.get("bundled") and g.get("source") == "missing"
)
counts = {key: data.get(f"gap_{key}", 0) for key in totals}
counts["missing"] -= emulator_bundled
counts["bundled"] = emulator_bundled
parts = [f"{count} {key}" for key, count in counts.items() if count]
status = ", ".join(parts) if parts else "OK"
systems = ", ".join(data["systems"])
@@ -840,13 +910,8 @@ def print_report(
total_gaps += gaps
for key in totals:
totals[key] += data.get(f"gap_{key}", 0)
emulator_bundled = sum(
1 for g in data["gap_details"]
if g.get("bundled") and g.get("source") == "missing"
)
totals[key] += counts[key]
bundled += emulator_bundled
totals["missing"] -= emulator_bundled
print(f"\n{'=' * 60}")
print(f"Total: {total_gaps} undeclared files across {scope}")
@@ -957,6 +1022,7 @@ def main() -> None:
profiles, declared, db, plat_data_dirs, data_names,
all_declared=declared_names(args.platforms_dir, selected, db),
standalone_cores=standalone_cores,
folded_slots=folded_declared_slots(args.platforms_dir, selected, db),
)
if args.json:
scripts/generate_pack.py
+53 -9
View File
@@ -94,6 +94,7 @@ from validation import (
_build_validation_index,
deliverable,
filter_files_by_mode,
with_archive_members,
)
yaml = require_yaml()
@@ -121,6 +122,20 @@ def _member_compression(arcname: str) -> int:
return zipfile.ZIP_DEFLATED
class PackRecord:
"""A pack build kept in memory: the paths it writes, no archive.
``members`` is every path written, ``files`` the ones the build counts as
the profile's files (archives and loose files, not trees or notes). The
pipeline builds every emulator pack this way and compares ``files`` with
what verify --emulator credits, without reading or compressing a byte.
"""
def __init__(self) -> None:
self.members: list[str] = []
self.files: list[str] = []
def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> None:
"""Add a file to the pack with metadata that does not depend on the build.
@@ -132,6 +147,9 @@ def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> Non
two consecutive builds. The content is streamed rather than read whole,
because some members are firmware images of several hundred megabytes.
"""
if isinstance(zf, PackRecord):
zf.members.append(arcname)
return
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
info.compress_type = _member_compression(arcname)
# The executable bit is the one permission worth carrying: the RetroDECK
@@ -158,6 +176,9 @@ def _write_generated_member(zf: zipfile.ZipFile, arcname: str, text: str) -> Non
a different hash on every run. Pinning them to the epoch the archive
rebuilder already uses makes a pack a function of its inputs alone.
"""
if isinstance(zf, PackRecord):
zf.members.append(arcname)
return
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o100644 << 16
@@ -934,6 +955,13 @@ def generate_pack(
) as tmp:
tmp_path = tmp.name
# The slot is the platform's file whether the download
# arrives or not: verify cannot fetch, and a core file
# filling it on a failed download made the two differ.
seen_destinations.add(dedup_key)
_register_path(dedup_key, seen_destinations, seen_parents)
if case_insensitive:
seen_lower.add(dedup_key.lower())
try:
if download_external(file_entry, tmp_path):
extract = file_entry.get("extract", False)
@@ -941,10 +969,6 @@ def generate_pack(
_extract_zip_to_archive(tmp_path, _flat(full_dest, base_dest, flatten), zf)
else:
_add_pack_member(zf, tmp_path, _flat(full_dest, base_dest, flatten))
seen_destinations.add(dedup_key)
_register_path(dedup_key, seen_destinations, seen_parents)
if case_insensitive:
seen_lower.add(dedup_key.lower())
file_status.setdefault(dedup_key, "ok")
total_files += 1
else:
@@ -1273,6 +1297,9 @@ def _add_zip_to_pack(
declare the hash of a ROM inside the ZIP (checkInsideZip), which a
content-preserving rebuild does not affect.
"""
if isinstance(target_zf, PackRecord):
target_zf.members.append(dest_path)
return
if (
file_entry
and not file_entry.get("zipped_file")
@@ -1414,6 +1441,7 @@ def generate_emulator_pack(
pack_name: str | None = None,
system_ids: list[str] | None = None,
final_dir: str | None = None,
record: PackRecord | None = None,
) -> str | None:
"""Generate a ZIP pack for specific emulator profiles.
@@ -1422,6 +1450,8 @@ def generate_emulator_pack(
the 255 bytes a file name may hold. `system_ids` keeps only the entries
of those systems, as a platform pack narrows core extras. `final_dir`
is where the caller moves the archive, the path the report names.
`record` builds the pack in memory instead (PackRecord): nothing is
written or read, and the paths are kept on it.
"""
all_profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
if zip_contents is None:
@@ -1446,7 +1476,8 @@ def generate_emulator_pack(
display_names, [n for n, _p in selected], f"{narrow_tags}_BIOS_Pack.zip"
)
zip_path = os.path.join(output_dir, zip_name)
os.makedirs(output_dir, exist_ok=True)
if record is None:
os.makedirs(output_dir, exist_ok=True)
region_drops: set[str] = (
emulator_region_drops(selected, standalone, regions, required_only, db)
@@ -1484,7 +1515,10 @@ def generate_emulator_pack(
platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms")
)
with _writing_pack(zip_path) as zf:
with (
contextlib.nullcontext(record) if record is not None
else _writing_pack(zip_path)
) as zf:
for emu_name, profile in sorted(selected):
pack_structure = profile.get("pack_structure")
files = expand_agnostic_entries(
@@ -1549,6 +1583,8 @@ def generate_emulator_pack(
seen_lower.add(archive_dest.lower())
packed_at[archive_dest.lower()] = local_path
total_files += 1
if record is not None:
record.files.append(archive_dest)
else:
report_missing(archive_name, archive_dest)
@@ -1608,7 +1644,7 @@ def generate_emulator_pack(
refused_path: str | None = None
for fe in alternatives:
local_path, status = resolve_file(
{**fe, "source_profile": emu_name},
{**with_archive_members(fe, files), "source_profile": emu_name},
db,
bios_dir,
zip_contents,
@@ -1676,12 +1712,16 @@ def generate_emulator_pack(
) as tmp:
tmp_path = tmp.name
try:
if download_external(fe, tmp_path):
# Built in memory, nothing is fetched: verify --emulator
# cannot fetch either and reports the file missing.
if record is None and download_external(fe, tmp_path):
_add_pack_member(zf, tmp_path, dest)
seen_destinations.add(dest)
_register_path(dest, seen_destinations, seen_parents)
seen_lower.add(dest.lower())
total_files += 1
if record is not None:
record.files.append(dest)
else:
missing_files.append(fe["name"])
finally:
@@ -1710,6 +1750,8 @@ def generate_emulator_pack(
seen_lower.add(dest.lower())
packed_at[dest.lower()] = local_path
total_files += 1
if record is not None:
record.files.append(dest)
# The trees after the files, as in a platform pack: a dump the
# profile declares wins over the copy a tree bundles at the same
@@ -1742,7 +1784,9 @@ def generate_emulator_pack(
)
# Remove empty ZIP (no files packed and no missing = nothing to ship)
if total_files + data_files == 0 and not (missing_files or rejected_files or instruction_files or unsourceable_files):
if record is None and total_files + data_files == 0 and not (
missing_files or rejected_files or instruction_files or unsourceable_files
):
os.unlink(zip_path)
# Report. The members of data trees are counted apart: verify --emulator
scripts/packextras.py
+128 -16
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
from collections.abc import Iterable
from common import PROFILE_IDENTITY_FIELDS, _norm_system_id
from common import archive_members
from common import archive_members, holds_required_members
from common import declared_root, expand_platform_declared_names, platform_declarations
from common import load_emulator_profiles
from common import resolution_is_hash_exact
@@ -209,7 +209,7 @@ def hidden_extras(
emulators_dir: str,
db: dict,
profiles: dict,
shipped: dict[str, str],
shipped: dict[str, str | None],
target_cores: set[str] | None = None,
region_drops: set[str] | None = None,
placed: set[str] | None = None,
@@ -229,7 +229,8 @@ def hidden_extras(
while the report counted it in the pack. A twin carrying the same bytes
serves the core on the filesystem that folds the names, so it is not a
file left out. ``shipped`` maps the full destination of each platform
file the pack carries to its local copy. ``placed``, when given,
file the pack carries to its local copy, or to None for a slot it keeps
without one (user_provided, external). ``placed``, when given,
receives the full destination of each held extra the pack carries.
``resolver`` is the builder's lookup. An extra it cannot resolve is
skipped and takes no path, as the builder skips it.
@@ -879,11 +880,17 @@ def _collect_emulator_extras(
undeclared = undeclared_report(
config, emulators_dir, db, emu_profiles, target_cores, include_all
)
# A destination two cores claim goes to the one that names its content:
# A destination two cores claim goes to the strongest proof of what the
# collection holds for it, then to the one that names its content:
# hypseus-singe asks for gamecontrollerdb.txt by name alone and took the
# BIOS root before Solarus, which pins its own by sha1. Profile order
# still decides between claims of the same kind.
# BIOS root before Solarus, which pins its own by sha1, and fbalpha2012's
# hashless hiscore.dat took RetroDECK's over mame2015's sha1. Profile
# order decides only between claims of equal proof.
undeclared = sorted(undeclared, key=lambda u: not _names_content(u))
rank = _ClaimRanks(
undeclared, placement, db, bool(config.get("case_insensitive_fs", False))
)
undeclared = rank.ordered(undeclared)
extras = []
seen_dests: set[str] = set(seen)
# Destination -> the extra that took it, to record later claimants on.
@@ -922,12 +929,18 @@ def _collect_emulator_extras(
claimants[(u.get("emulator", ""), u.get("name", ""), u.get("path") or "")] = (
sanitize_pack_path(dest)
)
extra = _report_extra(u, name, dest)
claims = [_report_extra(c, name, dest) for c in u.get("other_claims", [])]
# Where the builder meets each claim: the report lists the profiles in
# order, archives after the loose files.
for item in (extra, *claims):
item["claim_order"] = (1, 1 if u.get("archive") else 0, item["source_profile"])
candidates = [
_report_extra(c, name, dest) for c in (u, *u.get("other_claims", []))
]
# Where the builder meets each claim: the strongest proof first, the
# report's profile order between equals, archives after loose files.
for item in candidates:
item["claim_order"] = (
1, 1 if u.get("archive") else 0, -rank.of_extra(item, full_dest),
item["source_profile"],
)
candidates.sort(key=lambda item: item["claim_order"])
extra, claims = candidates[0], candidates[1:]
if not _take(extra, full_dest, seen_dests, taken):
for claim in claims:
_another_claim(taken.get(full_dest), claim)
@@ -1052,6 +1065,82 @@ def _collect_emulator_extras(
return extras
class _ClaimRanks:
"""How strongly each claim on a contested destination proves its file.
2 a declared hash the collection holds, 1 a path or every member the
core requires inside an archive, 0 a name, -1 a copy
contradicting the hash the claim declares, -2 nothing held. Resolved only
where two claims meet: a destination one core asks for has no contest.
"""
def __init__(
self, undeclared: list[dict], placement: ExtraPlacement, db: dict, fold: bool,
) -> None:
self._db = db
# Two spellings of one path are one file where the filesystem folds case.
self._key = (lambda dest: dest.lower()) if fold else (lambda dest: dest)
self._placed: dict[int, tuple[str, str] | None] = {}
counts: dict[str, int] = {}
for u in undeclared:
placed = (
placement.place(u, _extra_dest(u))
if u.get("in_repo") and not u.get("bundled") else None
)
self._placed[id(u)] = placed
if placed is not None:
key = self._key(placed[1])
counts[key] = counts.get(key, 0) + 1 + len(u.get("other_claims", []))
self._contested = {dest for dest, n in counts.items() if n > 1}
self._cache: dict[tuple, int] = {}
def of_extra(self, extra: dict, full_dest: str) -> int:
if self._key(full_dest) not in self._contested:
return 0
key = (extra.get("source_profile"), extra.get("source_name"),
extra.get("source_path"), full_dest)
if key not in self._cache:
path, status = resolve_local_file(
extra, self._db, dest_hint=extra.get("destination", "")
)
self._cache[key] = (
-2 if not path
else -1 if status == "hash_mismatch"
else 2 if resolution_is_hash_exact(status)
else 1 if status == "path_exact" or (
extra.get("members") and holds_required_members(path, extra["members"])
)
else 0
)
return self._cache[key]
def ordered(self, undeclared: list[dict]) -> list[dict]:
"""The entries in their order, the claims on one contested path
reordered among themselves by proof. Nothing else moves, so the
pack and the manifest keep their order."""
first: dict[str, int] = {}
keys: list[tuple[int, int, int]] = []
for position, u in enumerate(undeclared):
placed = self._placed.get(id(u))
if placed is None or self._key(placed[1]) not in self._contested:
keys.append((position, 0, position))
continue
group = first.setdefault(self._key(placed[1]), position)
keys.append((group, -self.of_entry(u), position))
order = sorted(range(len(undeclared)), key=keys.__getitem__)
return [undeclared[i] for i in order]
def of_entry(self, u: dict) -> int:
placed = self._placed.get(id(u))
if placed is None or self._key(placed[1]) not in self._contested:
return 0
name = u.get("archive") or u["name"]
return max(
self.of_extra(_report_extra(c, name, placed[0]), placed[1])
for c in (u, *u.get("other_claims", []))
)
def _names_content(entry: dict) -> bool:
"""Whether a report entry declares the bytes it wants, not only a name."""
return any(entry.get(field) for field in ("sha1", "md5", "sha256", "crc32"))
@@ -1316,11 +1405,12 @@ def emulator_region_drops(
"""
import region as region_mod
from packpaths import _resolve_destination
from validation import filter_files_by_mode
from validation import _build_validation_index, filter_files_by_mode
region_index = region_mod.build_region_index(dict(selected))
region_groups: dict[str, list[tuple[str, str]]] = {}
held: set[str] | None = set() if db is not None else None
validation_index = _build_validation_index(dict(selected)) if db is not None else {}
for emu_name, profile in sorted(selected):
# Decided over what the pack keeps, as on the platform side: an
# optional winner removed afterwards left its slot empty.
@@ -1332,13 +1422,35 @@ def emulator_region_drops(
region_groups.setdefault(group_id, []).append(
(dest, fe.get("name", ""))
)
if held is not None and resolve_local_file(
dict(fe, source_profile=emu_name), db, dest_hint=dest
)[0]:
if held is not None and _emulator_ships(
fe, emu_name, profile, db, dest, validation_index
):
held.add(dest)
return region_mod.resolve_region_drops(region_groups, region_index, regions, held)
def _emulator_ships(
file_entry: dict, emu_name: str, profile: dict, db: dict, dest: str,
validation_index: dict,
) -> bool:
"""Whether the emulator pack ships this entry: held, and not refused by
the core's own check, the test _ships applies on the platform side.
"""
from validation import validated_choice, with_archive_members
local, _status = resolve_local_file(
{**with_archive_members(file_entry, profile.get("files", [])),
"source_profile": emu_name},
db, dest_hint=dest,
)
if not local:
return False
_chosen, disagreement = validated_choice(
file_entry, local, db, validation_index, "bios", None, emulators={emu_name},
)
return disagreement is None
def _emulator_region_group(
emu_name: str, profile: dict, file_entry: dict, dest: str
) -> str:
scripts/pipeline.py
+174 -3
View File
@@ -21,7 +21,7 @@ Steps, numbered as the run prints them:
4b generate_pack.py --manifest install manifests
4c --manifest-targets per-target core lists
4d bundled_only.py --strict copies only an emulator install provides
5 consistency check verify counts == pack counts
5 consistency check verify == pack: counts, paths, every emulator pack
6 --verify-packs pack integrity, extraction and hashes
7 generate_readme.py README.md and CONTRIBUTING.md
8 generate_site.py MkDocs pages
@@ -41,13 +41,16 @@ Usage:
from __future__ import annotations
import argparse
import atexit
import contextlib
import importlib.util
import io
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path
@@ -382,6 +385,159 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
return all_ok
def parse_pack_paths(output: str) -> dict[str, str]:
"""The archive each full pack was written to, per pack label."""
paths: dict[str, str] = {}
current_label = ""
for line in output.splitlines():
m = re.match(r"Generating (?:shared )?pack for (.+)\.\.\.", line)
if m:
current_label = _full_pack_label(m.group(1).strip())
continue
written = re.match(r"^\s+(\S.*?\.zip): \d+ files packed", line)
if current_label and written:
paths[current_label] = written.group(1)
return paths
# Members the builder writes for the reader rather than for a core.
_PACK_NOTES = re.compile(r"(^README\.txt$|^manifest\.json$|(^|/)(INSTRUCTIONS|RENAMED)_[^/]*\.txt$)")
def _tree_members(platform: str, platforms_dir: str) -> dict[str, int]:
"""Archive path -> size of every file the platform's data trees can add."""
import generate_pack # noqa: PLC0415
from common import load_platform_config # noqa: PLC0415
from packpaths import _flat # noqa: PLC0415
config = load_platform_config(platform, platforms_dir)
base_dest = config.get("base_destination", "")
members: dict[str, int] = {}
with contextlib.redirect_stdout(io.StringIO()):
for src, full in generate_pack._data_directory_members(
config.get("systems", {}), load_data_dir_registry(platforms_dir),
platform, base_dest, False, set(), set(), set(),
):
members[_flat(full, base_dest, True)] = os.path.getsize(src)
return members
def _set_difference_lines(label: str, credited: set[str], written: set[str]) -> list[str]:
"""The paths one side has and the other lacks, ten of each at most."""
lines = []
for side, paths in (("verify only", credited - written), ("pack only", written - credited)):
if paths:
shown = ", ".join(sorted(paths)[:10])
more = f" (+{len(paths) - 10})" if len(paths) > 10 else ""
lines.append(f" {label}: {side} {len(paths)}: {shown}{more}")
return lines
def check_pack_destinations(
destinations_file: str, pack_output: str, platforms_dir: str = "platforms",
) -> bool:
"""Compare the paths verify credits with the members of each archive.
Equal counts hid a file credited on one side and another written on the
other. The trees and the notes are the archive's own: a tree member is
set aside when its path and size are the cached file's and verify does
not credit the path.
"""
import json # noqa: PLC0415
import zipfile # noqa: PLC0415
print("\n--- 5/8 consistency check: destinations ---")
try:
credited_by = json.loads(Path(destinations_file).read_text())
except (OSError, ValueError) as exc:
print(f" no destinations from verify ({exc})")
return False
paths = parse_pack_paths(pack_output)
if not paths:
print(" no archive named in the pack output")
return False
all_ok = True
for label, zip_path in sorted(paths.items()):
names = [n for n in credited_by if _match_key(n) & _match_key(label)]
if not names:
print(f" {label}: no verify destinations")
all_ok = False
continue
entry = credited_by[names[0]]
credited = set(entry["pack_destinations"])
trees = _tree_members(entry["platform"], platforms_dir)
with zipfile.ZipFile(zip_path) as zf:
written = {
info.filename for info in zf.infolist()
if not info.is_dir()
and not _PACK_NOTES.search(info.filename)
and not (
trees.get(info.filename) == info.file_size
and info.filename not in credited
)
}
lines = _set_difference_lines(label, credited, written)
if lines:
print("\n".join(lines))
all_ok = False
else:
print(f" {label}: {len(credited)} paths, the same on both sides")
return all_ok
def check_emulator_packs(
emulators_dir: str = "emulators", platforms_dir: str = "platforms",
db_path: str = "database.json",
) -> bool:
"""Build every emulator pack in memory and compare it with verify --emulator.
Each profile in each build it has (libretro, standalone): the paths the
pack fills with the profile's files against the ones verify credits. No
archive is written and no file read, so all of them take seconds.
"""
import generate_pack # noqa: PLC0415
import verify # noqa: PLC0415
from common import ( # noqa: PLC0415
ProfileSelectionError,
build_zip_contents_index,
load_database,
load_emulator_profiles,
select_emulator_profiles,
)
print("\n--- 5/8 consistency check: emulator packs ---")
db = load_database(db_path)
zip_contents = build_zip_contents_index(db)
profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
all_ok = True
compared = 0
for name in sorted(profiles):
for standalone in (False, True):
try:
select_emulator_profiles([name], profiles, standalone)
except ProfileSelectionError:
continue
record = generate_pack.PackRecord()
with contextlib.redirect_stdout(io.StringIO()):
generate_pack.generate_emulator_pack(
[name], emulators_dir, db, "bios", "", standalone, zip_contents,
offline=True, platforms_dir=platforms_dir, record=record,
)
report = verify.verify_emulator(
[name], emulators_dir, db, standalone, platforms_dir=platforms_dir,
)
compared += 1
label = f"{name}{' --standalone' if standalone else ''}"
lines = _set_difference_lines(
label, set(report["pack_destinations"]), set(record.files)
)
if lines:
print("\n".join(lines))
all_ok = False
print(f" {compared} emulator packs compared, {'all equal' if all_ok else 'differences above'}")
return all_ok
class _Skipped:
"""A step that did not run. Truthy, so it never fails the pipeline, and
distinct, so the summary does not report it as work that was done."""
@@ -681,8 +837,19 @@ def main() -> None:
# exclusive.
hold_data_dirs(held, load_data_dir_registry("platforms"))
# Step 3: Verify
verify_cmd = [sys.executable, "scripts/verify.py", "--all", "--strict"]
# Step 3: Verify. The paths each pack holds by this report go to a file
# step 5 compares with the archives.
scratch = Path(__file__).resolve().parent.parent / "tmp"
scratch.mkdir(exist_ok=True)
fd, destinations_file = tempfile.mkstemp(
prefix="verify-destinations.", suffix=".json", dir=scratch
)
os.close(fd)
atexit.register(Path(destinations_file).unlink, missing_ok=True)
verify_cmd = [
sys.executable, "scripts/verify.py", "--all", "--strict",
"--destinations", destinations_file,
]
if args.include_archived:
verify_cmd.append("--include-archived")
if args.target:
@@ -833,6 +1000,10 @@ def main() -> None:
results["consistency"] = SKIPPED
elif pack_output and verify_output:
ok = check_consistency(verify_output, pack_output)
# Counts can agree while the paths differ, both ways at once.
ok = check_pack_destinations(destinations_file, pack_output) and ok
ok = check_emulator_packs() and ok
print(f"--- 5/8 consistency check, counts and paths: {'OK' if ok else 'FAILED'} ---")
results["consistency"] = ok
all_ok = all_ok and ok
else:
scripts/slots.py
+56 -22
View File
@@ -1,8 +1,8 @@
"""Who decides what goes at a destination.
A destination is a slot. Two layers can claim it: the platform YAML, scraped
from what the frontend declares and therefore able to carry an upstream error,
and the emulator profile, read from the emulator's own source. Until this
from what the frontend declares, which can pin another file than the emulator
reads, and the emulator profile, read from the emulator's own source. Until this
module existed nothing compared the two: the builder resolves the platform
entry first and drops the profile entry on a bare filename match, so a profile
that names the right file for a slot could never win and never even be heard.
@@ -37,7 +37,7 @@ from common import (
variant_base_name,
)
from packresolve import preferred_entries
from validation import outside_gap_scope
from validation import outside_gap_scope, with_archive_members
if TYPE_CHECKING:
from packextras import ExtraPlacement
@@ -241,7 +241,10 @@ def profile_claims(
full = placed[1]
# The owner's own copy is the one the builder ships.
local, status = resolve_local_file(
{**entry, "source_profile": emu_name},
{
**with_archive_members(entry, profile.get("files") or []),
"source_profile": emu_name,
},
db,
zip_contents,
dest_hint=dest,
@@ -326,6 +329,29 @@ def _archive_claims(
return claims
def checked_by_path(mode: str, entry: dict, rule: str = "") -> bool:
"""Whether the frontend checks this declaration by its path alone.
Whether a frontend reads the bytes is a property of the platform, but
what it compares them with is the entry's: Batocera skips an empty md5
(batocera-systems:1088), and EmuDeck declares neocdz.zip with no hash at
all. Such an entry is a path, in an md5 platform as in an existence one.
A member pinned inside an archive is content.
"""
if not nativemode.reads_file_contents(mode):
return True
if entry.get("zipped_file"):
return False
if rule == nativemode.SIZE_AND_ANY_HASH:
fields: tuple[str, ...] = ("md5", "sha1", "crc32")
else:
fields = (nativemode.digest_algorithm(mode) or "",)
return not any(
parse_md5_list(entry.get(name)) if name == "md5" else entry.get(name)
for name in fields
)
def _claim_rank(claim: Claim) -> int:
"""How strongly a platform claim speaks for what the pack ships."""
if resolution_is_hash_exact(claim.status):
@@ -379,10 +405,10 @@ def _slot_claims(
# matches, asserts nothing about content. Where the frontend never reads
# the bytes, a proven profile claim then decides: RetroArch's neocdz.zip
# pins an md5 nobody holds, and the primary shipped without the 000-lo.lo
# geolith and FBNeo read inside it.
path_only = not nativemode.reads_file_contents(
config.get("verification_mode", "existence")
)
# geolith and FBNeo read inside it. Decided per entry: EmuDeck, an md5
# platform, declares the same archive with no hash.
mode = config.get("verification_mode", "existence")
rule = config.get("hash_rule", "")
by_slot: dict[str, list[Claim]] = {}
for claim in profile_claims(
profiles, db, base_dest, standalone_cores, zip_contents, data_dir_registry,
@@ -398,7 +424,9 @@ def _slot_claims(
):
# A size proves a claim only against a file of another size.
continue
if not platform.is_proven and not (path_only and platform.local_path):
if not platform.is_proven and not (
platform.local_path and checked_by_path(mode, platform.entry, rule)
):
continue
by_slot.setdefault(key, []).append(claim)
@@ -515,8 +543,8 @@ def slot_choices(
recorded whole in the install manifest, so a target narrowed from the
full manifest gets the bytes the target's own pack carries.
"""
if nativemode.reads_file_contents(config.get("verification_mode", "existence")):
return {}
mode = config.get("verification_mode", "existence")
rule = config.get("hash_rule", "")
platform_profiles = {
name: profiles[name] for name in resolve_platform_cores(config, profiles)
}
@@ -532,6 +560,9 @@ def slot_choices(
choices: dict[str, list[tuple[str, str | None]]] = {}
for key, claims in sorted(by_slot.items()):
platform = by_dest[key]
# Where the frontend compares a hash, the platform's file stays.
if not checked_by_path(mode, platform.entry, rule):
continue
agree = [_satisfies(c, platform, db) for c in claims]
agreeing = [c for c, ok in zip(claims, agree) if ok]
others = sorted(
@@ -570,7 +601,9 @@ def _strongest(claims: list[Claim]) -> Claim:
return max(claims, key=lambda claim: (_claim_rank(claim), -claims.index(claim)))
def arbitrate(conflict: Conflict, mode: str, addressee: str = "platform") -> Decision:
def arbitrate(
conflict: Conflict, mode: str, addressee: str = "platform", rule: str = "",
) -> Decision:
"""Decide a contested destination for the pack being built.
A pack answers to whoever asked for it. A platform pack must leave the
@@ -581,13 +614,13 @@ def arbitrate(conflict: Conflict, mode: str, addressee: str = "platform") -> Dec
In ``existence`` mode the frontend only looks for a path, so the emulator's
file satisfies both sides at once and there is nothing to trade away. In a
content-checking mode the two answers cannot share one path, and the pack's
addressee decides. The loss is reported rather than absorbed, because the
cause is an upstream declaration that needs fixing at its source.
addressee decides. The loss is reported rather than absorbed: only the
platform's own list can change what its frontend accepts there.
"""
profile = _strongest(conflict.profile_claims)
if addressee == "emulator":
return Decision(conflict, profile, ADDRESSEE)
if not nativemode.reads_file_contents(mode):
if checked_by_path(mode, conflict.platform_claim.entry, rule):
return Decision(conflict, profile, SERVES_BOTH)
return Decision(conflict, conflict.platform_claim, FRONTEND_CHECKS_CONTENT)
@@ -626,10 +659,11 @@ def format_decision(decision: Decision) -> str:
f"{conflict.destination}: serve {decision.winner.local_path}, "
"the pack answers to the emulator"
)
expected = ", ".join(conflict.emulators) or "the profile"
return (
f"{conflict.destination}: keep {decision.winner.local_path}, the frontend "
f"verifies content and would reject "
f"{conflict.profile_claims[0].local_path}; upstream declaration is wrong"
f"{conflict.profile_claims[0].local_path}, the file {expected} reads"
)
@@ -659,8 +693,8 @@ def find_collisions(
One path holds one file, so whichever declaration the builder reaches
first decides in silence. Declaring an archive several times to name its
inner ROMs is the documented zipped_file pattern and resolves to one
archive. Two declarations landing on two files is a contradiction the
upstream list carries. RetroDECK aims a PC-88 disk subsystem ROM and a
archive. Two declarations landing on two files name two files for one
path. RetroDECK aims a PC-88 disk subsystem ROM and a
CoCo disk ROM at one bios/disk.rom.
"""
fold = bool(config.get("case_insensitive_fs", False))
@@ -974,7 +1008,7 @@ def main() -> int:
mode = config.get("verification_mode", "existence")
print(f"{platform}: {len(conflicts)} contradicted [{mode}]")
for conflict in conflicts:
decision = arbitrate(conflict, mode)
decision = arbitrate(conflict, mode, rule=config.get("hash_rule", ""))
fixable += decision.serves_both
print(f" {format_decision(decision)}")
total = sum(len(c) for c in found.values())
@@ -987,13 +1021,13 @@ def main() -> int:
doubled = sum(len(c) for c in collided.values())
print(
f"\n{total} contested destinations. {fixable} the pack settles on "
f"its own, {total - fixable} rest on an upstream declaration. "
f"its own, {total - fixable} follow the platform's list. "
f"{doubled} destinations declared twice."
)
if not args.strict and (total or doubled):
print(
"Reported, not failed: the remainder needs the upstream list "
"corrected, which no build can do. Use --strict to gate on them."
"Reported, not failed: the remainder changes when the platform's "
"list does, which no build can do. Use --strict to gate on them."
)
return 1 if ((found or collided) and args.strict) else 0
scripts/validation.py
+34 -1
View File
@@ -13,6 +13,7 @@ from collections.abc import Callable, Iterator, Mapping
from common import (
adler32_of,
archive_members,
compute_hashes,
name_match_size_ok,
sanitize_pack_path,
@@ -464,11 +465,25 @@ def outside_gap_scope(file_entry: dict, is_standalone: bool) -> bool:
)
def case_folded_slots(
config: dict, declarations: Mapping[str, list[tuple[int | None, str]]],
) -> frozenset[str]:
"""The slots a platform whose filesystem folds case fills, any spelling.
On RetroBat the 32X_G_BIOS.BIN it declares is the file BlastEm opens as
32X_G_BIOS.bin. On Lakka bios.col and BIOS.col are two files: nothing.
"""
if not config.get("case_insensitive_fs", False):
return frozenset()
return frozenset(where for places in declarations.values() for _size, where in places)
def declaration_settles(
file_entry: dict,
fname: str,
declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str],
dest: str = "",
folded_slots: frozenset[str] = frozenset(),
) -> bool:
"""Whether a platform declaration meets a core's requirement.
@@ -485,12 +500,16 @@ def declaration_settles(
carries no slot or size and answers by name.
Read by verify and by the gap report, which settled on the name alone
and left DOSBox's SC-55 ROMs off the gaps page.
and left DOSBox's SC-55 ROMs off the gaps page. ``folded_slots``
(case_folded_slots) are slots a declaration fills in another spelling on
a filesystem that folds case: the core opens that file there.
"""
if file_entry.get("archive") in declared_names:
return True
slot = sanitize_pack_path(dest or fname).lower()
directory = slot if dest.endswith("/") else slot.rpartition("/")[0]
if not dest.endswith("/") and slot in folded_slots:
return True
for name in (fname, *file_entry.get("aliases", [])):
if name not in declared_names:
continue
@@ -512,6 +531,20 @@ def declaration_settles(
return False
def with_archive_members(entry: dict, files: list[dict]) -> dict:
"""A loose entry resolved as the archive its profile reads members from.
gngeopi declares neogeo.zip and, apart, the ROMs it opens inside it. The
members decide which same-named copy is the archive: resolved by name
alone, the loose entry took another copy than the archive pass and the
profile blocked its own archive.
"""
if entry.get("archive"):
return entry
members = archive_members(entry.get("name", ""), files)
return {**entry, "members": members} if members else entry
def opens_by_name(file_entry: dict, profile: dict, standalone: bool = False) -> bool:
"""Whether the code opens this entry by its name.
scripts/verify.py
+118 -10
View File
@@ -30,6 +30,7 @@ import json
import os
import sys
from collections.abc import Mapping
from pathlib import Path
import slots
@@ -85,6 +86,7 @@ from nativemode import (
)
from validation import (
baseline_verdict,
case_folded_slots,
declaration_settles,
frontend_digest_matches,
existence_discrepancy,
@@ -100,6 +102,7 @@ from validation import (
installed_with_emulator,
unread_on,
outside_gap_scope,
with_archive_members,
)
DEFAULT_DB = "database.json"
@@ -442,6 +445,7 @@ def _candidate_verdict(
include_all: bool,
declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str],
dest: str = "",
folded_slots: frozenset[str] = frozenset(),
) -> str:
"""Whether a profile entry can be a gap, and whether it is settled.
@@ -457,7 +461,7 @@ def _candidate_verdict(
if outside_gap_scope(file_entry, is_standalone):
return "skip"
if not include_all and declaration_settles(
file_entry, fname, declared_names, dest
file_entry, fname, declared_names, dest, folded_slots
):
return "settled"
return "keep"
@@ -501,6 +505,7 @@ def find_undeclared_files(
"""
if declared_names is None:
declared_names = platform_declarations(config, db)
folded_slots = case_folded_slots(config, declared_names)
# Whether the builder drops a file whose local copy contradicts its
# declared hash, which decides if such a copy counts as held here.
@@ -590,7 +595,8 @@ def find_undeclared_files(
if duplicate and (first is None or first["profile"] == emu_name):
continue
verdict = _candidate_verdict(
f, fname, is_standalone, include_all, declared_names, dest
f, fname, is_standalone, include_all, declared_names, dest,
folded_slots,
)
if verdict == "settled":
seen_files.add(seen_key)
@@ -707,6 +713,9 @@ def find_undeclared_files(
}
if f.get("contents"):
entry["contents"] = f["contents"]
members = with_archive_members(f, profile.get("files", [])).get("members")
if members:
entry["members"] = members
if launcher_layer:
launcher_candidates.append((emu_name, is_standalone, f, entry))
if duplicate:
@@ -879,7 +888,7 @@ def _mark_hidden(
emulators_dir: str,
db: dict,
emu_profiles: dict | None,
shipped: dict[str, str],
shipped: dict[str, str | None],
target_cores: set[str] | None,
region_drops: set[str] | None = None,
report: list[dict] | None = None,
@@ -911,6 +920,44 @@ def _mark_hidden(
return placed
def _builder_placement(
packed_order: list[tuple[str, str, str | None]], case_insensitive: bool,
) -> tuple[dict[str, str], dict[str, str | None]]:
"""The platform files the builder writes, and every path it keeps.
The builder walks the systems in name order and skips a path that a
written file blocks as a directory, or the reverse: RetroArch declares
SGB1.sfc and SGB1.sfc/program.rom, and only the first is in the pack.
An entry without a destination (EmuDeck's system:md5 whitelist) is
judged but takes no path. Returns (path -> local copy written there,
path -> copy or None for each slot kept, a user_provided or external one
included).
"""
from packpaths import _has_path_conflict, _register_path
shipped: dict[str, str] = {}
occupied: dict[str, str | None] = {}
seen: set[str] = set()
folded: set[str] = set()
parents: set[str] = set()
for _sys_id, full, local in sorted(packed_order, key=lambda item: item[0]):
# EmuDeck's system:md5 entries name no file: the builder writes none.
if not full or full.endswith("/"):
continue
if (
full in occupied
or (case_insensitive and full.lower() in folded)
or _has_path_conflict(full, seen, parents)
):
continue
_register_path(full, seen, parents)
folded.add(full.lower())
occupied[full] = local
if local is not None:
shipped[full] = local
return shipped, occupied
def _dest_key(file_entry: dict, sys_id: str) -> str:
"""The path a declaration occupies, as the builder keys its pack."""
dest = sanitize_pack_path(file_entry.get("destination", file_entry.get("name", "")))
@@ -1148,7 +1195,10 @@ def verify_platform(
verify_systems, db, base_dest, zip_contents
)
shipped: dict[str, str] = {}
# (system, full destination, local copy) of each declaration the builder
# writes, and None for one whose slot it keeps without a file from the
# collection (user_provided, external). Placed below in the builder's order.
packed_order: list[tuple[str, str, str | None]] = []
shipped_keys: set[str] = set()
results_at: dict[str, list[dict]] = {}
for sys_id, system in verify_systems.items():
@@ -1172,6 +1222,12 @@ def verify_platform(
if stored is not None:
result = stored
local_path = None
# The builder keeps the slot: instructions for a user_provided
# file, the download for an external one, whether it arrives
# or not. No core file goes there.
packed_order.append((
sys_id, f"{base_dest}/{destination}" if base_dest else destination, None,
))
elif not reads_file_contents(mode):
result = verify_entry_existence(
file_entry,
@@ -1216,10 +1272,11 @@ def verify_platform(
file_entry, local_path, resolve_status, mode, config.get("hash_rule", "")
)[1] is None
):
shipped.setdefault(
packed_order.append((
sys_id,
f"{base_dest}/{destination}" if base_dest else destination,
local_path,
)
))
shipped_keys.add(_dest_key(file_entry, sys_id))
if _twin_unmet(
result, held_by, file_entry, sys_id,
@@ -1252,6 +1309,10 @@ def verify_platform(
):
file_severity[dest] = sev
shipped, occupied = _builder_placement(
packed_order, bool(config.get("case_insensitive_fs", False))
)
# Cross-reference undeclared files, called exactly as the builder calls
# it: its declarations, not the enriched ones (a core file the platform
# also declares under another name is packed at the core's own path, 21
@@ -1283,7 +1344,7 @@ def verify_platform(
placed_by: dict[str, tuple[str, str, str]] = {}
placed = _mark_hidden(
config, undeclared, emulators_dir, db, emu_profiles, shipped, target_cores,
config, undeclared, emulators_dir, db, emu_profiles, occupied, target_cores,
region_drops, report_for_builder,
ExtraResolver(
db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index,
@@ -1363,10 +1424,23 @@ def verify_platform(
)
total = len(file_status)
# The paths a pack built from this report holds, as written in the ZIP
# (base_destination stripped, as the builder flattens it). The pipeline
# compares this set with the archive: equal counts can hide one file
# credited here and another written there.
from packpaths import _flat
pack_destinations = sorted(
{_flat(dest, base_dest, True) for dest in shipped} | {
_flat(dest, base_dest, True) for dest in placed
}
)
return {
"platform": platform,
"verification_mode": mode,
"total_files": total,
"pack_destinations": pack_destinations,
"severity_counts": counts,
"status_counts": status_counts,
"excluded_files": excluded,
@@ -1590,8 +1664,11 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
core_missing_req = len(req_not_in_repo) + len(req_hle_not_in_repo)
core_missing_opt = len(opt_not_in_repo)
# Required gaps are always named below, the optional ones on request.
listed = "" if verbose or not core_missing_opt else " (--verbose lists them)"
print(
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, {core_missing_opt} optional missing"
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, "
f"{core_missing_opt} optional missing{listed}"
)
if bundled:
names = ", ".join(sorted({u["name"] for u in bundled}))
@@ -1603,6 +1680,9 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
_print_undeclared_entry(u, "MISSING (required)", verbose)
for u in req_hle_not_in_repo:
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose)
if verbose:
for u in opt_not_in_repo:
_print_undeclared_entry(u, "MISSING (optional)", verbose)
if game_data:
gd_missing = [u for u in game_data if not u["in_repo"]]
@@ -1612,7 +1692,11 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
)
gd_unplaced = [u for u in game_data if u["in_repo"] and not u.get("in_pack", True)]
if gd_missing or gd_packed or gd_unplaced:
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing")
listed = "" if verbose or not gd_missing else " (--verbose lists them)"
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}")
if verbose:
for u in gd_missing:
_print_undeclared_entry(u, "MISSING (game data)", verbose)
if gd_unplaced:
print(
f" Game data held but not packed: {len(gd_unplaced)} "
@@ -1951,7 +2035,7 @@ def verify_emulator(
dest_hint = file_entry.get("path", "")
local_path, resolve_status = resolve_local_file(
{**file_entry, "source_profile": emu_name},
{**with_archive_members(file_entry, files), "source_profile": emu_name},
db,
zip_contents,
dest_hint=dest_hint,
@@ -2151,6 +2235,11 @@ def verify_emulator(
"emulators": [n for n, _ in selected],
"verification_mode": label,
"total_files": total,
# The paths the emulator pack fills with the profile's files, for the
# pipeline's comparison with the pack built in memory.
"pack_destinations": sorted(
dest for dest, status in file_status.items() if status == Status.OK
),
"severity_counts": counts,
"status_counts": status_counts,
"details": details,
@@ -2331,6 +2420,7 @@ def _refuse_listing_narrowings(
("--json", args.json),
("--strict", args.strict),
("--verbose", args.verbose),
("--destinations", args.destinations),
):
if on:
parser.error(f"{flag} is incompatible with {listing}")
@@ -2346,6 +2436,10 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
"--target requires --platform or --all"),
(args.target and (args.emulator or args.system),
"--target is incompatible with --emulator and --system"),
(args.destinations and not (args.platform or args.all),
"--destinations requires --platform or --all"),
(args.destinations and getattr(args, "region", None),
"--destinations describes the full pack and is incompatible with --region"),
):
if refused:
parser.error(message)
@@ -2423,6 +2517,11 @@ def _main(held: contextlib.ExitStack) -> None:
help="Show emulator ground truth details",
)
parser.add_argument("--json", action="store_true", help="JSON output")
parser.add_argument(
"--destinations", metavar="FILE",
help="With --platform or --all, also write the paths each pack holds "
"according to this report to FILE (JSON), for comparison with the archive",
)
parser.add_argument(
"--strict",
action="store_true",
@@ -2555,6 +2654,15 @@ def _main(held: contextlib.ExitStack) -> None:
if len(group_results) > 1:
print(_overall_summary(group_results))
if args.destinations:
held = {}
for (result, names), (group, _rep) in zip(group_results, groups):
for name, key in zip(names, group):
held[name] = {
"platform": key, "pack_destinations": result["pack_destinations"],
}
Path(args.destinations).write_text(json.dumps(held, indent=1, sort_keys=True))
if args.json:
for r in all_results.values():
r["details"] = _json_details(r["details"], args.verbose)