chore: merge audit fixes (integrity)

# Conflicts:
#	install/batocera.json
#	install/emudeck.json
#	install/lakka.json
#	install/recalbox.json
#	install/retroarch.json
#	install/retrobat.json
#	install/retrodeck.json
#	install/retropie.json
#	install/rocknix.json
#	install/romm.json
#	scripts/generate_pack.py
#	wiki/tools.md
This commit is contained in:
Abdessamad Derraz committed 2026-10-21 06:48:42 +02:00
commit 378a9e4da5
19 files changed
+1199 -101

No files matched your search

+88 -22
View File
@@ -31,6 +31,7 @@ from common import (
get_mame_clone_map,
list_emulator_profiles,
list_registered_platforms,
load_data_dir_registry,
load_database,
load_emulator_profiles,
load_platform_config,
@@ -44,7 +45,7 @@ from common import (
select_emulator_profiles,
)
from hashing import parse_hash_list
from validation import declaration_settles, outside_gap_scope
from validation import case_folded_slots, declaration_settles, outside_gap_scope
yaml = require_yaml()
@@ -105,6 +106,25 @@ def declared_names(
)
def folded_declared_slots(
platforms_dir: str, platforms: list[str] | None, db: dict
) -> frozenset[str]:
"""The slots platforms whose filesystem folds case fill, any spelling.
RetroBat's 32X_G_BIOS.BIN is the file BlastEm opens as 32X_G_BIOS.bin,
and the exact-case name test called it undeclared.
"""
from common import platform_declarations
slots: set[str] = set()
for platform_name in platforms or list_registered_platforms(
platforms_dir, include_archived=True
):
config = load_platform_config(platform_name, platforms_dir)
slots |= case_folded_slots(config, platform_declarations(config, db))
return frozenset(slots)
def _build_supplemental_index(
data_root: str = "data", bios_root: str = "bios"
) -> set[str]:
@@ -178,21 +198,28 @@ def _resolve_source(
or ``None`` (not available anywhere).
A name hit whose size contradicts a size the emulator verifies is not the
file, so it does not count as available.
file, so it does not count as available. Nor is one whose bytes carry
none of the hashes the entry declares: a name never wins over a declared
hash, and zquestclassic's ending.mid read as held because Hydra Castle
Labyrinth has another file of that name.
"""
declared = _declared_content(file_entry or {})
def _name_hit(name: str) -> bool:
"""Whether the files indexed under *name* can be this entry."""
def _can_be(sha1s: list | str) -> bool:
if not file_entry or db_files is None:
return True
sha1s = by_name.get(name) or []
if not isinstance(sha1s, list):
sha1s = [sha1s]
return any(
name_match_size_ok(file_entry, db_files.get(sha1, {}).get("size"))
and _record_carries(sha1, db_files.get(sha1, {}), declared)
for sha1 in sha1s
)
def _name_hit(name: str) -> bool:
"""Whether the files indexed under *name* can be this entry."""
return _can_be(by_name.get(name) or [])
# bios/ via database.json by_name
if fname in by_name and _name_hit(fname):
return "bios"
@@ -209,7 +236,7 @@ def _resolve_source(
):
return "bios"
# bios/ via by_path_suffix (regional variants)
if by_path_suffix and fname in by_path_suffix:
if by_path_suffix and fname in by_path_suffix and _can_be(by_path_suffix[fname]):
return "bios"
# bios/ under the canonical MAME set name, as resolve_local_file does:
# a renamed archive is held once, under the name the dedup kept.
@@ -252,6 +279,23 @@ def _data_hit(
_HASH_FIELDS = ("sha1", "md5", "sha256", "crc32")
def _record_carries(sha1: str, record: dict, declared: dict[str, set[str]]) -> bool:
"""Whether a database record has one of the declared hashes, or none is declared."""
if not declared:
return True
actual = {
"sha1": sha1.lower(),
**{field: str(record.get(field) or "").lower() for field in ("md5", "sha256", "crc32")},
}
return any(
actual[field] and (
actual[field] in values
or (field == "md5" and any(len(v) < 32 and actual[field].startswith(v) for v in values))
)
for field, values in declared.items()
)
def _declared_content(file_entry: dict) -> dict[str, set[str]]:
"""The hashes an entry declares, lowercased, by algorithm."""
declared: dict[str, set[str]] = {}
@@ -326,9 +370,21 @@ class FileConsumers:
file by the evidence the pack uses, so a homonym is nobody's file.
"""
def __init__(self, db: dict, zip_contents: dict | None = None) -> None:
def __init__(
self, db: dict, zip_contents: dict | None = None,
data_dir_registry: dict | None = None,
) -> None:
self._db = db
self._zip_contents = zip_contents
# The builder resolves with the data trees: Dolphin's codehandler.bin
# comes from data/dolphin-sys, and without them the name step gave it
# Dolphin-MMJR's copy, which the site then said Dolphin reads.
self._data_dir_registry = (
data_dir_registry if data_dir_registry is not None
else load_data_dir_registry(
str(Path(__file__).resolve().parent.parent / DEFAULT_PLATFORMS_DIR)
)
)
self._sha1_by_path = {
path: sha1
for sha1, entry in db.get("files", {}).items()
@@ -362,7 +418,8 @@ class FileConsumers:
if owner:
entry = {**entry, "source_profile": owner}
path, status = resolve_local_file(
entry, self._db, self._zip_contents, dest_hint=dest
entry, self._db, self._zip_contents, dest_hint=dest,
data_dir_registry=self._data_dir_registry,
)
if not path or status in ("not_found", "hash_mismatch"):
return None
@@ -470,6 +527,7 @@ def entry_source(f: dict, index: dict) -> str | None:
{"indexes": {"by_path_suffix": by_path_suffix}}, path_field or fname
):
return "bios"
found = None
for candidate in [fname, *([path_field] if path_field != fname else []),
*(f.get("aliases") or [])]:
if not candidate:
@@ -478,12 +536,15 @@ def entry_source(f: dict, index: dict) -> str | None:
candidate, by_name, by_name_lower, data_names, by_path_suffix, f, db_files,
index.get("bios_dir"),
)
if source is not None:
if source == "bios":
return source
found = found or source
def values(field: str) -> list[str]:
return parse_hash_list(f.get(field))
# The declared content anywhere in bios/ comes before a data/ copy, as in
# the resolver: the hash steps run before the data directories.
held = (
any(index["by_md5"].get(v) for v in parse_md5_list(f.get("md5")))
or any(v in db_files for v in values("sha1"))
@@ -491,7 +552,7 @@ def entry_source(f: dict, index: dict) -> str | None:
or any(v in index["by_sha256"] for v in values("sha256"))
or any(index["by_crc32"].get(v) for v in values("crc32"))
)
return "bios" if held else None
return "bios" if held else found
def _resolve_archive_source(
@@ -629,7 +690,9 @@ def _cross_reference_profile(
# Check platform declaration: by name, by an alias the core answers
# to (RetroArch lists amiberry's cd32.rom as kick40060.CD32), or by
# archive.
in_platform = declaration_settles(f, fname, platform_names, effective_path)
in_platform = declaration_settles(
f, fname, platform_names, effective_path, index.get("folded_slots", frozenset())
)
if in_platform:
seen_files.add(seen_key)
@@ -736,6 +799,7 @@ def cross_reference(
data_names: set[str] | None = None,
all_declared: dict[str, list[tuple[int | None, str]]] | None = None,
standalone_cores: set[str] | None = None,
folded_slots: frozenset[str] = frozenset(),
) -> dict:
"""Compare emulator profiles against platform declarations.
@@ -772,6 +836,7 @@ def cross_reference(
"db_files": db_files,
"data_names": data_names,
"all_declared": all_declared,
"folded_slots": folded_slots,
"bios_dir": db.get("bios_dir"),
}
for emu_name, profile in profiles.items():
@@ -807,11 +872,16 @@ def print_report(
if gaps == 0 and not show_clean:
continue
parts = []
for key in ("bios", "data", "large_file", "missing"):
count = data.get(f"gap_{key}", 0)
if count:
parts.append(f"{count} {key}")
# Counted once, for the heading and the total alike: a file bundled
# with its emulator is not one to source.
emulator_bundled = sum(
1 for g in data["gap_details"]
if g.get("bundled") and g.get("source") == "missing"
)
counts = {key: data.get(f"gap_{key}", 0) for key in totals}
counts["missing"] -= emulator_bundled
counts["bundled"] = emulator_bundled
parts = [f"{count} {key}" for key, count in counts.items() if count]
status = ", ".join(parts) if parts else "OK"
systems = ", ".join(data["systems"])
@@ -840,13 +910,8 @@ def print_report(
total_gaps += gaps
for key in totals:
totals[key] += data.get(f"gap_{key}", 0)
emulator_bundled = sum(
1 for g in data["gap_details"]
if g.get("bundled") and g.get("source") == "missing"
)
totals[key] += counts[key]
bundled += emulator_bundled
totals["missing"] -= emulator_bundled
print(f"\n{'=' * 60}")
print(f"Total: {total_gaps} undeclared files across {scope}")
@@ -957,6 +1022,7 @@ def main() -> None:
profiles, declared, db, plat_data_dirs, data_names,
all_declared=declared_names(args.platforms_dir, selected, db),
standalone_cores=standalone_cores,
folded_slots=folded_declared_slots(args.platforms_dir, selected, db),
)
if args.json:
+53 -9
View File
@@ -94,6 +94,7 @@ from validation import (
_build_validation_index,
deliverable,
filter_files_by_mode,
with_archive_members,
)
yaml = require_yaml()
@@ -121,6 +122,20 @@ def _member_compression(arcname: str) -> int:
return zipfile.ZIP_DEFLATED
class PackRecord:
"""A pack build kept in memory: the paths it writes, no archive.
``members`` is every path written, ``files`` the ones the build counts as
the profile's files (archives and loose files, not trees or notes). The
pipeline builds every emulator pack this way and compares ``files`` with
what verify --emulator credits, without reading or compressing a byte.
"""
def __init__(self) -> None:
self.members: list[str] = []
self.files: list[str] = []
def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> None:
"""Add a file to the pack with metadata that does not depend on the build.
@@ -132,6 +147,9 @@ def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> Non
two consecutive builds. The content is streamed rather than read whole,
because some members are firmware images of several hundred megabytes.
"""
if isinstance(zf, PackRecord):
zf.members.append(arcname)
return
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
info.compress_type = _member_compression(arcname)
# The executable bit is the one permission worth carrying: the RetroDECK
@@ -158,6 +176,9 @@ def _write_generated_member(zf: zipfile.ZipFile, arcname: str, text: str) -> Non
a different hash on every run. Pinning them to the epoch the archive
rebuilder already uses makes a pack a function of its inputs alone.
"""
if isinstance(zf, PackRecord):
zf.members.append(arcname)
return
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
info.compress_type = zipfile.ZIP_DEFLATED
info.external_attr = 0o100644 << 16
@@ -934,6 +955,13 @@ def generate_pack(
) as tmp:
tmp_path = tmp.name
# The slot is the platform's file whether the download
# arrives or not: verify cannot fetch, and a core file
# filling it on a failed download made the two differ.
seen_destinations.add(dedup_key)
_register_path(dedup_key, seen_destinations, seen_parents)
if case_insensitive:
seen_lower.add(dedup_key.lower())
try:
if download_external(file_entry, tmp_path):
extract = file_entry.get("extract", False)
@@ -941,10 +969,6 @@ def generate_pack(
_extract_zip_to_archive(tmp_path, _flat(full_dest, base_dest, flatten), zf)
else:
_add_pack_member(zf, tmp_path, _flat(full_dest, base_dest, flatten))
seen_destinations.add(dedup_key)
_register_path(dedup_key, seen_destinations, seen_parents)
if case_insensitive:
seen_lower.add(dedup_key.lower())
file_status.setdefault(dedup_key, "ok")
total_files += 1
else:
@@ -1273,6 +1297,9 @@ def _add_zip_to_pack(
declare the hash of a ROM inside the ZIP (checkInsideZip), which a
content-preserving rebuild does not affect.
"""
if isinstance(target_zf, PackRecord):
target_zf.members.append(dest_path)
return
if (
file_entry
and not file_entry.get("zipped_file")
@@ -1414,6 +1441,7 @@ def generate_emulator_pack(
pack_name: str | None = None,
system_ids: list[str] | None = None,
final_dir: str | None = None,
record: PackRecord | None = None,
) -> str | None:
"""Generate a ZIP pack for specific emulator profiles.
@@ -1422,6 +1450,8 @@ def generate_emulator_pack(
the 255 bytes a file name may hold. `system_ids` keeps only the entries
of those systems, as a platform pack narrows core extras. `final_dir`
is where the caller moves the archive, the path the report names.
`record` builds the pack in memory instead (PackRecord): nothing is
written or read, and the paths are kept on it.
"""
all_profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
if zip_contents is None:
@@ -1446,7 +1476,8 @@ def generate_emulator_pack(
display_names, [n for n, _p in selected], f"{narrow_tags}_BIOS_Pack.zip"
)
zip_path = os.path.join(output_dir, zip_name)
os.makedirs(output_dir, exist_ok=True)
if record is None:
os.makedirs(output_dir, exist_ok=True)
region_drops: set[str] = (
emulator_region_drops(selected, standalone, regions, required_only, db)
@@ -1484,7 +1515,10 @@ def generate_emulator_pack(
platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms")
)
with _writing_pack(zip_path) as zf:
with (
contextlib.nullcontext(record) if record is not None
else _writing_pack(zip_path)
) as zf:
for emu_name, profile in sorted(selected):
pack_structure = profile.get("pack_structure")
files = expand_agnostic_entries(
@@ -1549,6 +1583,8 @@ def generate_emulator_pack(
seen_lower.add(archive_dest.lower())
packed_at[archive_dest.lower()] = local_path
total_files += 1
if record is not None:
record.files.append(archive_dest)
else:
report_missing(archive_name, archive_dest)
@@ -1608,7 +1644,7 @@ def generate_emulator_pack(
refused_path: str | None = None
for fe in alternatives:
local_path, status = resolve_file(
{**fe, "source_profile": emu_name},
{**with_archive_members(fe, files), "source_profile": emu_name},
db,
bios_dir,
zip_contents,
@@ -1676,12 +1712,16 @@ def generate_emulator_pack(
) as tmp:
tmp_path = tmp.name
try:
if download_external(fe, tmp_path):
# Built in memory, nothing is fetched: verify --emulator
# cannot fetch either and reports the file missing.
if record is None and download_external(fe, tmp_path):
_add_pack_member(zf, tmp_path, dest)
seen_destinations.add(dest)
_register_path(dest, seen_destinations, seen_parents)
seen_lower.add(dest.lower())
total_files += 1
if record is not None:
record.files.append(dest)
else:
missing_files.append(fe["name"])
finally:
@@ -1710,6 +1750,8 @@ def generate_emulator_pack(
seen_lower.add(dest.lower())
packed_at[dest.lower()] = local_path
total_files += 1
if record is not None:
record.files.append(dest)
# The trees after the files, as in a platform pack: a dump the
# profile declares wins over the copy a tree bundles at the same
@@ -1742,7 +1784,9 @@ def generate_emulator_pack(
)
# Remove empty ZIP (no files packed and no missing = nothing to ship)
if total_files + data_files == 0 and not (missing_files or rejected_files or instruction_files or unsourceable_files):
if record is None and total_files + data_files == 0 and not (
missing_files or rejected_files or instruction_files or unsourceable_files
):
os.unlink(zip_path)
# Report. The members of data trees are counted apart: verify --emulator
+128 -16
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
from collections.abc import Iterable
from common import PROFILE_IDENTITY_FIELDS, _norm_system_id
from common import archive_members
from common import archive_members, holds_required_members
from common import declared_root, expand_platform_declared_names, platform_declarations
from common import load_emulator_profiles
from common import resolution_is_hash_exact
@@ -209,7 +209,7 @@ def hidden_extras(
emulators_dir: str,
db: dict,
profiles: dict,
shipped: dict[str, str],
shipped: dict[str, str | None],
target_cores: set[str] | None = None,
region_drops: set[str] | None = None,
placed: set[str] | None = None,
@@ -229,7 +229,8 @@ def hidden_extras(
while the report counted it in the pack. A twin carrying the same bytes
serves the core on the filesystem that folds the names, so it is not a
file left out. ``shipped`` maps the full destination of each platform
file the pack carries to its local copy. ``placed``, when given,
file the pack carries to its local copy, or to None for a slot it keeps
without one (user_provided, external). ``placed``, when given,
receives the full destination of each held extra the pack carries.
``resolver`` is the builder's lookup. An extra it cannot resolve is
skipped and takes no path, as the builder skips it.
@@ -879,11 +880,17 @@ def _collect_emulator_extras(
undeclared = undeclared_report(
config, emulators_dir, db, emu_profiles, target_cores, include_all
)
# A destination two cores claim goes to the one that names its content:
# A destination two cores claim goes to the strongest proof of what the
# collection holds for it, then to the one that names its content:
# hypseus-singe asks for gamecontrollerdb.txt by name alone and took the
# BIOS root before Solarus, which pins its own by sha1. Profile order
# still decides between claims of the same kind.
# BIOS root before Solarus, which pins its own by sha1, and fbalpha2012's
# hashless hiscore.dat took RetroDECK's over mame2015's sha1. Profile
# order decides only between claims of equal proof.
undeclared = sorted(undeclared, key=lambda u: not _names_content(u))
rank = _ClaimRanks(
undeclared, placement, db, bool(config.get("case_insensitive_fs", False))
)
undeclared = rank.ordered(undeclared)
extras = []
seen_dests: set[str] = set(seen)
# Destination -> the extra that took it, to record later claimants on.
@@ -922,12 +929,18 @@ def _collect_emulator_extras(
claimants[(u.get("emulator", ""), u.get("name", ""), u.get("path") or "")] = (
sanitize_pack_path(dest)
)
extra = _report_extra(u, name, dest)
claims = [_report_extra(c, name, dest) for c in u.get("other_claims", [])]
# Where the builder meets each claim: the report lists the profiles in
# order, archives after the loose files.
for item in (extra, *claims):
item["claim_order"] = (1, 1 if u.get("archive") else 0, item["source_profile"])
candidates = [
_report_extra(c, name, dest) for c in (u, *u.get("other_claims", []))
]
# Where the builder meets each claim: the strongest proof first, the
# report's profile order between equals, archives after loose files.
for item in candidates:
item["claim_order"] = (
1, 1 if u.get("archive") else 0, -rank.of_extra(item, full_dest),
item["source_profile"],
)
candidates.sort(key=lambda item: item["claim_order"])
extra, claims = candidates[0], candidates[1:]
if not _take(extra, full_dest, seen_dests, taken):
for claim in claims:
_another_claim(taken.get(full_dest), claim)
@@ -1052,6 +1065,82 @@ def _collect_emulator_extras(
return extras
class _ClaimRanks:
"""How strongly each claim on a contested destination proves its file.
2 a declared hash the collection holds, 1 a path or every member the
core requires inside an archive, 0 a name, -1 a copy
contradicting the hash the claim declares, -2 nothing held. Resolved only
where two claims meet: a destination one core asks for has no contest.
"""
def __init__(
self, undeclared: list[dict], placement: ExtraPlacement, db: dict, fold: bool,
) -> None:
self._db = db
# Two spellings of one path are one file where the filesystem folds case.
self._key = (lambda dest: dest.lower()) if fold else (lambda dest: dest)
self._placed: dict[int, tuple[str, str] | None] = {}
counts: dict[str, int] = {}
for u in undeclared:
placed = (
placement.place(u, _extra_dest(u))
if u.get("in_repo") and not u.get("bundled") else None
)
self._placed[id(u)] = placed
if placed is not None:
key = self._key(placed[1])
counts[key] = counts.get(key, 0) + 1 + len(u.get("other_claims", []))
self._contested = {dest for dest, n in counts.items() if n > 1}
self._cache: dict[tuple, int] = {}
def of_extra(self, extra: dict, full_dest: str) -> int:
if self._key(full_dest) not in self._contested:
return 0
key = (extra.get("source_profile"), extra.get("source_name"),
extra.get("source_path"), full_dest)
if key not in self._cache:
path, status = resolve_local_file(
extra, self._db, dest_hint=extra.get("destination", "")
)
self._cache[key] = (
-2 if not path
else -1 if status == "hash_mismatch"
else 2 if resolution_is_hash_exact(status)
else 1 if status == "path_exact" or (
extra.get("members") and holds_required_members(path, extra["members"])
)
else 0
)
return self._cache[key]
def ordered(self, undeclared: list[dict]) -> list[dict]:
"""The entries in their order, the claims on one contested path
reordered among themselves by proof. Nothing else moves, so the
pack and the manifest keep their order."""
first: dict[str, int] = {}
keys: list[tuple[int, int, int]] = []
for position, u in enumerate(undeclared):
placed = self._placed.get(id(u))
if placed is None or self._key(placed[1]) not in self._contested:
keys.append((position, 0, position))
continue
group = first.setdefault(self._key(placed[1]), position)
keys.append((group, -self.of_entry(u), position))
order = sorted(range(len(undeclared)), key=keys.__getitem__)
return [undeclared[i] for i in order]
def of_entry(self, u: dict) -> int:
placed = self._placed.get(id(u))
if placed is None or self._key(placed[1]) not in self._contested:
return 0
name = u.get("archive") or u["name"]
return max(
self.of_extra(_report_extra(c, name, placed[0]), placed[1])
for c in (u, *u.get("other_claims", []))
)
def _names_content(entry: dict) -> bool:
"""Whether a report entry declares the bytes it wants, not only a name."""
return any(entry.get(field) for field in ("sha1", "md5", "sha256", "crc32"))
@@ -1316,11 +1405,12 @@ def emulator_region_drops(
"""
import region as region_mod
from packpaths import _resolve_destination
from validation import filter_files_by_mode
from validation import _build_validation_index, filter_files_by_mode
region_index = region_mod.build_region_index(dict(selected))
region_groups: dict[str, list[tuple[str, str]]] = {}
held: set[str] | None = set() if db is not None else None
validation_index = _build_validation_index(dict(selected)) if db is not None else {}
for emu_name, profile in sorted(selected):
# Decided over what the pack keeps, as on the platform side: an
# optional winner removed afterwards left its slot empty.
@@ -1332,13 +1422,35 @@ def emulator_region_drops(
region_groups.setdefault(group_id, []).append(
(dest, fe.get("name", ""))
)
if held is not None and resolve_local_file(
dict(fe, source_profile=emu_name), db, dest_hint=dest
)[0]:
if held is not None and _emulator_ships(
fe, emu_name, profile, db, dest, validation_index
):
held.add(dest)
return region_mod.resolve_region_drops(region_groups, region_index, regions, held)
def _emulator_ships(
file_entry: dict, emu_name: str, profile: dict, db: dict, dest: str,
validation_index: dict,
) -> bool:
"""Whether the emulator pack ships this entry: held, and not refused by
the core's own check, the test _ships applies on the platform side.
"""
from validation import validated_choice, with_archive_members
local, _status = resolve_local_file(
{**with_archive_members(file_entry, profile.get("files", [])),
"source_profile": emu_name},
db, dest_hint=dest,
)
if not local:
return False
_chosen, disagreement = validated_choice(
file_entry, local, db, validation_index, "bios", None, emulators={emu_name},
)
return disagreement is None
def _emulator_region_group(
emu_name: str, profile: dict, file_entry: dict, dest: str
) -> str:
+174 -3
View File
@@ -21,7 +21,7 @@ Steps, numbered as the run prints them:
4b generate_pack.py --manifest install manifests
4c --manifest-targets per-target core lists
4d bundled_only.py --strict copies only an emulator install provides
5 consistency check verify counts == pack counts
5 consistency check verify == pack: counts, paths, every emulator pack
6 --verify-packs pack integrity, extraction and hashes
7 generate_readme.py README.md and CONTRIBUTING.md
8 generate_site.py MkDocs pages
@@ -41,13 +41,16 @@ Usage:
from __future__ import annotations
import argparse
import atexit
import contextlib
import importlib.util
import io
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path
@@ -382,6 +385,159 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
return all_ok
def parse_pack_paths(output: str) -> dict[str, str]:
"""The archive each full pack was written to, per pack label."""
paths: dict[str, str] = {}
current_label = ""
for line in output.splitlines():
m = re.match(r"Generating (?:shared )?pack for (.+)\.\.\.", line)
if m:
current_label = _full_pack_label(m.group(1).strip())
continue
written = re.match(r"^\s+(\S.*?\.zip): \d+ files packed", line)
if current_label and written:
paths[current_label] = written.group(1)
return paths
# Members the builder writes for the reader rather than for a core.
_PACK_NOTES = re.compile(r"(^README\.txt$|^manifest\.json$|(^|/)(INSTRUCTIONS|RENAMED)_[^/]*\.txt$)")
def _tree_members(platform: str, platforms_dir: str) -> dict[str, int]:
"""Archive path -> size of every file the platform's data trees can add."""
import generate_pack # noqa: PLC0415
from common import load_platform_config # noqa: PLC0415
from packpaths import _flat # noqa: PLC0415
config = load_platform_config(platform, platforms_dir)
base_dest = config.get("base_destination", "")
members: dict[str, int] = {}
with contextlib.redirect_stdout(io.StringIO()):
for src, full in generate_pack._data_directory_members(
config.get("systems", {}), load_data_dir_registry(platforms_dir),
platform, base_dest, False, set(), set(), set(),
):
members[_flat(full, base_dest, True)] = os.path.getsize(src)
return members
def _set_difference_lines(label: str, credited: set[str], written: set[str]) -> list[str]:
"""The paths one side has and the other lacks, ten of each at most."""
lines = []
for side, paths in (("verify only", credited - written), ("pack only", written - credited)):
if paths:
shown = ", ".join(sorted(paths)[:10])
more = f" (+{len(paths) - 10})" if len(paths) > 10 else ""
lines.append(f" {label}: {side} {len(paths)}: {shown}{more}")
return lines
def check_pack_destinations(
destinations_file: str, pack_output: str, platforms_dir: str = "platforms",
) -> bool:
"""Compare the paths verify credits with the members of each archive.
Equal counts hid a file credited on one side and another written on the
other. The trees and the notes are the archive's own: a tree member is
set aside when its path and size are the cached file's and verify does
not credit the path.
"""
import json # noqa: PLC0415
import zipfile # noqa: PLC0415
print("\n--- 5/8 consistency check: destinations ---")
try:
credited_by = json.loads(Path(destinations_file).read_text())
except (OSError, ValueError) as exc:
print(f" no destinations from verify ({exc})")
return False
paths = parse_pack_paths(pack_output)
if not paths:
print(" no archive named in the pack output")
return False
all_ok = True
for label, zip_path in sorted(paths.items()):
names = [n for n in credited_by if _match_key(n) & _match_key(label)]
if not names:
print(f" {label}: no verify destinations")
all_ok = False
continue
entry = credited_by[names[0]]
credited = set(entry["pack_destinations"])
trees = _tree_members(entry["platform"], platforms_dir)
with zipfile.ZipFile(zip_path) as zf:
written = {
info.filename for info in zf.infolist()
if not info.is_dir()
and not _PACK_NOTES.search(info.filename)
and not (
trees.get(info.filename) == info.file_size
and info.filename not in credited
)
}
lines = _set_difference_lines(label, credited, written)
if lines:
print("\n".join(lines))
all_ok = False
else:
print(f" {label}: {len(credited)} paths, the same on both sides")
return all_ok
def check_emulator_packs(
emulators_dir: str = "emulators", platforms_dir: str = "platforms",
db_path: str = "database.json",
) -> bool:
"""Build every emulator pack in memory and compare it with verify --emulator.
Each profile in each build it has (libretro, standalone): the paths the
pack fills with the profile's files against the ones verify credits. No
archive is written and no file read, so all of them take seconds.
"""
import generate_pack # noqa: PLC0415
import verify # noqa: PLC0415
from common import ( # noqa: PLC0415
ProfileSelectionError,
build_zip_contents_index,
load_database,
load_emulator_profiles,
select_emulator_profiles,
)
print("\n--- 5/8 consistency check: emulator packs ---")
db = load_database(db_path)
zip_contents = build_zip_contents_index(db)
profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
all_ok = True
compared = 0
for name in sorted(profiles):
for standalone in (False, True):
try:
select_emulator_profiles([name], profiles, standalone)
except ProfileSelectionError:
continue
record = generate_pack.PackRecord()
with contextlib.redirect_stdout(io.StringIO()):
generate_pack.generate_emulator_pack(
[name], emulators_dir, db, "bios", "", standalone, zip_contents,
offline=True, platforms_dir=platforms_dir, record=record,
)
report = verify.verify_emulator(
[name], emulators_dir, db, standalone, platforms_dir=platforms_dir,
)
compared += 1
label = f"{name}{' --standalone' if standalone else ''}"
lines = _set_difference_lines(
label, set(report["pack_destinations"]), set(record.files)
)
if lines:
print("\n".join(lines))
all_ok = False
print(f" {compared} emulator packs compared, {'all equal' if all_ok else 'differences above'}")
return all_ok
class _Skipped:
"""A step that did not run. Truthy, so it never fails the pipeline, and
distinct, so the summary does not report it as work that was done."""
@@ -681,8 +837,19 @@ def main() -> None:
# exclusive.
hold_data_dirs(held, load_data_dir_registry("platforms"))
# Step 3: Verify
verify_cmd = [sys.executable, "scripts/verify.py", "--all", "--strict"]
# Step 3: Verify. The paths each pack holds by this report go to a file
# step 5 compares with the archives.
scratch = Path(__file__).resolve().parent.parent / "tmp"
scratch.mkdir(exist_ok=True)
fd, destinations_file = tempfile.mkstemp(
prefix="verify-destinations.", suffix=".json", dir=scratch
)
os.close(fd)
atexit.register(Path(destinations_file).unlink, missing_ok=True)
verify_cmd = [
sys.executable, "scripts/verify.py", "--all", "--strict",
"--destinations", destinations_file,
]
if args.include_archived:
verify_cmd.append("--include-archived")
if args.target:
@@ -833,6 +1000,10 @@ def main() -> None:
results["consistency"] = SKIPPED
elif pack_output and verify_output:
ok = check_consistency(verify_output, pack_output)
# Counts can agree while the paths differ, both ways at once.
ok = check_pack_destinations(destinations_file, pack_output) and ok
ok = check_emulator_packs() and ok
print(f"--- 5/8 consistency check, counts and paths: {'OK' if ok else 'FAILED'} ---")
results["consistency"] = ok
all_ok = all_ok and ok
else:
+56 -22
View File
@@ -1,8 +1,8 @@
"""Who decides what goes at a destination.
A destination is a slot. Two layers can claim it: the platform YAML, scraped
from what the frontend declares and therefore able to carry an upstream error,
and the emulator profile, read from the emulator's own source. Until this
from what the frontend declares, which can pin another file than the emulator
reads, and the emulator profile, read from the emulator's own source. Until this
module existed nothing compared the two: the builder resolves the platform
entry first and drops the profile entry on a bare filename match, so a profile
that names the right file for a slot could never win and never even be heard.
@@ -37,7 +37,7 @@ from common import (
variant_base_name,
)
from packresolve import preferred_entries
from validation import outside_gap_scope
from validation import outside_gap_scope, with_archive_members
if TYPE_CHECKING:
from packextras import ExtraPlacement
@@ -241,7 +241,10 @@ def profile_claims(
full = placed[1]
# The owner's own copy is the one the builder ships.
local, status = resolve_local_file(
{**entry, "source_profile": emu_name},
{
**with_archive_members(entry, profile.get("files") or []),
"source_profile": emu_name,
},
db,
zip_contents,
dest_hint=dest,
@@ -326,6 +329,29 @@ def _archive_claims(
return claims
def checked_by_path(mode: str, entry: dict, rule: str = "") -> bool:
"""Whether the frontend checks this declaration by its path alone.
Whether a frontend reads the bytes is a property of the platform, but
what it compares them with is the entry's: Batocera skips an empty md5
(batocera-systems:1088), and EmuDeck declares neocdz.zip with no hash at
all. Such an entry is a path, in an md5 platform as in an existence one.
A member pinned inside an archive is content.
"""
if not nativemode.reads_file_contents(mode):
return True
if entry.get("zipped_file"):
return False
if rule == nativemode.SIZE_AND_ANY_HASH:
fields: tuple[str, ...] = ("md5", "sha1", "crc32")
else:
fields = (nativemode.digest_algorithm(mode) or "",)
return not any(
parse_md5_list(entry.get(name)) if name == "md5" else entry.get(name)
for name in fields
)
def _claim_rank(claim: Claim) -> int:
"""How strongly a platform claim speaks for what the pack ships."""
if resolution_is_hash_exact(claim.status):
@@ -379,10 +405,10 @@ def _slot_claims(
# matches, asserts nothing about content. Where the frontend never reads
# the bytes, a proven profile claim then decides: RetroArch's neocdz.zip
# pins an md5 nobody holds, and the primary shipped without the 000-lo.lo
# geolith and FBNeo read inside it.
path_only = not nativemode.reads_file_contents(
config.get("verification_mode", "existence")
)
# geolith and FBNeo read inside it. Decided per entry: EmuDeck, an md5
# platform, declares the same archive with no hash.
mode = config.get("verification_mode", "existence")
rule = config.get("hash_rule", "")
by_slot: dict[str, list[Claim]] = {}
for claim in profile_claims(
profiles, db, base_dest, standalone_cores, zip_contents, data_dir_registry,
@@ -398,7 +424,9 @@ def _slot_claims(
):
# A size proves a claim only against a file of another size.
continue
if not platform.is_proven and not (path_only and platform.local_path):
if not platform.is_proven and not (
platform.local_path and checked_by_path(mode, platform.entry, rule)
):
continue
by_slot.setdefault(key, []).append(claim)
@@ -515,8 +543,8 @@ def slot_choices(
recorded whole in the install manifest, so a target narrowed from the
full manifest gets the bytes the target's own pack carries.
"""
if nativemode.reads_file_contents(config.get("verification_mode", "existence")):
return {}
mode = config.get("verification_mode", "existence")
rule = config.get("hash_rule", "")
platform_profiles = {
name: profiles[name] for name in resolve_platform_cores(config, profiles)
}
@@ -532,6 +560,9 @@ def slot_choices(
choices: dict[str, list[tuple[str, str | None]]] = {}
for key, claims in sorted(by_slot.items()):
platform = by_dest[key]
# Where the frontend compares a hash, the platform's file stays.
if not checked_by_path(mode, platform.entry, rule):
continue
agree = [_satisfies(c, platform, db) for c in claims]
agreeing = [c for c, ok in zip(claims, agree) if ok]
others = sorted(
@@ -570,7 +601,9 @@ def _strongest(claims: list[Claim]) -> Claim:
return max(claims, key=lambda claim: (_claim_rank(claim), -claims.index(claim)))
def arbitrate(conflict: Conflict, mode: str, addressee: str = "platform") -> Decision:
def arbitrate(
conflict: Conflict, mode: str, addressee: str = "platform", rule: str = "",
) -> Decision:
"""Decide a contested destination for the pack being built.
A pack answers to whoever asked for it. A platform pack must leave the
@@ -581,13 +614,13 @@ def arbitrate(conflict: Conflict, mode: str, addressee: str = "platform") -> Dec
In ``existence`` mode the frontend only looks for a path, so the emulator's
file satisfies both sides at once and there is nothing to trade away. In a
content-checking mode the two answers cannot share one path, and the pack's
addressee decides. The loss is reported rather than absorbed, because the
cause is an upstream declaration that needs fixing at its source.
addressee decides. The loss is reported rather than absorbed: only the
platform's own list can change what its frontend accepts there.
"""
profile = _strongest(conflict.profile_claims)
if addressee == "emulator":
return Decision(conflict, profile, ADDRESSEE)
if not nativemode.reads_file_contents(mode):
if checked_by_path(mode, conflict.platform_claim.entry, rule):
return Decision(conflict, profile, SERVES_BOTH)
return Decision(conflict, conflict.platform_claim, FRONTEND_CHECKS_CONTENT)
@@ -626,10 +659,11 @@ def format_decision(decision: Decision) -> str:
f"{conflict.destination}: serve {decision.winner.local_path}, "
"the pack answers to the emulator"
)
expected = ", ".join(conflict.emulators) or "the profile"
return (
f"{conflict.destination}: keep {decision.winner.local_path}, the frontend "
f"verifies content and would reject "
f"{conflict.profile_claims[0].local_path}; upstream declaration is wrong"
f"{conflict.profile_claims[0].local_path}, the file {expected} reads"
)
@@ -659,8 +693,8 @@ def find_collisions(
One path holds one file, so whichever declaration the builder reaches
first decides in silence. Declaring an archive several times to name its
inner ROMs is the documented zipped_file pattern and resolves to one
archive. Two declarations landing on two files is a contradiction the
upstream list carries. RetroDECK aims a PC-88 disk subsystem ROM and a
archive. Two declarations landing on two files name two files for one
path. RetroDECK aims a PC-88 disk subsystem ROM and a
CoCo disk ROM at one bios/disk.rom.
"""
fold = bool(config.get("case_insensitive_fs", False))
@@ -974,7 +1008,7 @@ def main() -> int:
mode = config.get("verification_mode", "existence")
print(f"{platform}: {len(conflicts)} contradicted [{mode}]")
for conflict in conflicts:
decision = arbitrate(conflict, mode)
decision = arbitrate(conflict, mode, rule=config.get("hash_rule", ""))
fixable += decision.serves_both
print(f" {format_decision(decision)}")
total = sum(len(c) for c in found.values())
@@ -987,13 +1021,13 @@ def main() -> int:
doubled = sum(len(c) for c in collided.values())
print(
f"\n{total} contested destinations. {fixable} the pack settles on "
f"its own, {total - fixable} rest on an upstream declaration. "
f"its own, {total - fixable} follow the platform's list. "
f"{doubled} destinations declared twice."
)
if not args.strict and (total or doubled):
print(
"Reported, not failed: the remainder needs the upstream list "
"corrected, which no build can do. Use --strict to gate on them."
"Reported, not failed: the remainder changes when the platform's "
"list does, which no build can do. Use --strict to gate on them."
)
return 1 if ((found or collided) and args.strict) else 0
+34 -1
View File
@@ -13,6 +13,7 @@ from collections.abc import Callable, Iterator, Mapping
from common import (
adler32_of,
archive_members,
compute_hashes,
name_match_size_ok,
sanitize_pack_path,
@@ -464,11 +465,25 @@ def outside_gap_scope(file_entry: dict, is_standalone: bool) -> bool:
)
def case_folded_slots(
config: dict, declarations: Mapping[str, list[tuple[int | None, str]]],
) -> frozenset[str]:
"""The slots a platform whose filesystem folds case fills, any spelling.
On RetroBat the 32X_G_BIOS.BIN it declares is the file BlastEm opens as
32X_G_BIOS.bin. On Lakka bios.col and BIOS.col are two files: nothing.
"""
if not config.get("case_insensitive_fs", False):
return frozenset()
return frozenset(where for places in declarations.values() for _size, where in places)
def declaration_settles(
file_entry: dict,
fname: str,
declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str],
dest: str = "",
folded_slots: frozenset[str] = frozenset(),
) -> bool:
"""Whether a platform declaration meets a core's requirement.
@@ -485,12 +500,16 @@ def declaration_settles(
carries no slot or size and answers by name.
Read by verify and by the gap report, which settled on the name alone
and left DOSBox's SC-55 ROMs off the gaps page.
and left DOSBox's SC-55 ROMs off the gaps page. ``folded_slots``
(case_folded_slots) are slots a declaration fills in another spelling on
a filesystem that folds case: the core opens that file there.
"""
if file_entry.get("archive") in declared_names:
return True
slot = sanitize_pack_path(dest or fname).lower()
directory = slot if dest.endswith("/") else slot.rpartition("/")[0]
if not dest.endswith("/") and slot in folded_slots:
return True
for name in (fname, *file_entry.get("aliases", [])):
if name not in declared_names:
continue
@@ -512,6 +531,20 @@ def declaration_settles(
return False
def with_archive_members(entry: dict, files: list[dict]) -> dict:
"""A loose entry resolved as the archive its profile reads members from.
gngeopi declares neogeo.zip and, apart, the ROMs it opens inside it. The
members decide which same-named copy is the archive: resolved by name
alone, the loose entry took another copy than the archive pass and the
profile blocked its own archive.
"""
if entry.get("archive"):
return entry
members = archive_members(entry.get("name", ""), files)
return {**entry, "members": members} if members else entry
def opens_by_name(file_entry: dict, profile: dict, standalone: bool = False) -> bool:
"""Whether the code opens this entry by its name.
+118 -10
View File
@@ -30,6 +30,7 @@ import json
import os
import sys
from collections.abc import Mapping
from pathlib import Path
import slots
@@ -85,6 +86,7 @@ from nativemode import (
)
from validation import (
baseline_verdict,
case_folded_slots,
declaration_settles,
frontend_digest_matches,
existence_discrepancy,
@@ -100,6 +102,7 @@ from validation import (
installed_with_emulator,
unread_on,
outside_gap_scope,
with_archive_members,
)
DEFAULT_DB = "database.json"
@@ -442,6 +445,7 @@ def _candidate_verdict(
include_all: bool,
declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str],
dest: str = "",
folded_slots: frozenset[str] = frozenset(),
) -> str:
"""Whether a profile entry can be a gap, and whether it is settled.
@@ -457,7 +461,7 @@ def _candidate_verdict(
if outside_gap_scope(file_entry, is_standalone):
return "skip"
if not include_all and declaration_settles(
file_entry, fname, declared_names, dest
file_entry, fname, declared_names, dest, folded_slots
):
return "settled"
return "keep"
@@ -501,6 +505,7 @@ def find_undeclared_files(
"""
if declared_names is None:
declared_names = platform_declarations(config, db)
folded_slots = case_folded_slots(config, declared_names)
# Whether the builder drops a file whose local copy contradicts its
# declared hash, which decides if such a copy counts as held here.
@@ -590,7 +595,8 @@ def find_undeclared_files(
if duplicate and (first is None or first["profile"] == emu_name):
continue
verdict = _candidate_verdict(
f, fname, is_standalone, include_all, declared_names, dest
f, fname, is_standalone, include_all, declared_names, dest,
folded_slots,
)
if verdict == "settled":
seen_files.add(seen_key)
@@ -707,6 +713,9 @@ def find_undeclared_files(
}
if f.get("contents"):
entry["contents"] = f["contents"]
members = with_archive_members(f, profile.get("files", [])).get("members")
if members:
entry["members"] = members
if launcher_layer:
launcher_candidates.append((emu_name, is_standalone, f, entry))
if duplicate:
@@ -879,7 +888,7 @@ def _mark_hidden(
emulators_dir: str,
db: dict,
emu_profiles: dict | None,
shipped: dict[str, str],
shipped: dict[str, str | None],
target_cores: set[str] | None,
region_drops: set[str] | None = None,
report: list[dict] | None = None,
@@ -911,6 +920,44 @@ def _mark_hidden(
return placed
def _builder_placement(
packed_order: list[tuple[str, str, str | None]], case_insensitive: bool,
) -> tuple[dict[str, str], dict[str, str | None]]:
"""The platform files the builder writes, and every path it keeps.
The builder walks the systems in name order and skips a path that a
written file blocks as a directory, or the reverse: RetroArch declares
SGB1.sfc and SGB1.sfc/program.rom, and only the first is in the pack.
An entry without a destination (EmuDeck's system:md5 whitelist) is
judged but takes no path. Returns (path -> local copy written there,
path -> copy or None for each slot kept, a user_provided or external one
included).
"""
from packpaths import _has_path_conflict, _register_path
shipped: dict[str, str] = {}
occupied: dict[str, str | None] = {}
seen: set[str] = set()
folded: set[str] = set()
parents: set[str] = set()
for _sys_id, full, local in sorted(packed_order, key=lambda item: item[0]):
# EmuDeck's system:md5 entries name no file: the builder writes none.
if not full or full.endswith("/"):
continue
if (
full in occupied
or (case_insensitive and full.lower() in folded)
or _has_path_conflict(full, seen, parents)
):
continue
_register_path(full, seen, parents)
folded.add(full.lower())
occupied[full] = local
if local is not None:
shipped[full] = local
return shipped, occupied
def _dest_key(file_entry: dict, sys_id: str) -> str:
"""The path a declaration occupies, as the builder keys its pack."""
dest = sanitize_pack_path(file_entry.get("destination", file_entry.get("name", "")))
@@ -1148,7 +1195,10 @@ def verify_platform(
verify_systems, db, base_dest, zip_contents
)
shipped: dict[str, str] = {}
# (system, full destination, local copy) of each declaration the builder
# writes, and None for one whose slot it keeps without a file from the
# collection (user_provided, external). Placed below in the builder's order.
packed_order: list[tuple[str, str, str | None]] = []
shipped_keys: set[str] = set()
results_at: dict[str, list[dict]] = {}
for sys_id, system in verify_systems.items():
@@ -1172,6 +1222,12 @@ def verify_platform(
if stored is not None:
result = stored
local_path = None
# The builder keeps the slot: instructions for a user_provided
# file, the download for an external one, whether it arrives
# or not. No core file goes there.
packed_order.append((
sys_id, f"{base_dest}/{destination}" if base_dest else destination, None,
))
elif not reads_file_contents(mode):
result = verify_entry_existence(
file_entry,
@@ -1216,10 +1272,11 @@ def verify_platform(
file_entry, local_path, resolve_status, mode, config.get("hash_rule", "")
)[1] is None
):
shipped.setdefault(
packed_order.append((
sys_id,
f"{base_dest}/{destination}" if base_dest else destination,
local_path,
)
))
shipped_keys.add(_dest_key(file_entry, sys_id))
if _twin_unmet(
result, held_by, file_entry, sys_id,
@@ -1252,6 +1309,10 @@ def verify_platform(
):
file_severity[dest] = sev
shipped, occupied = _builder_placement(
packed_order, bool(config.get("case_insensitive_fs", False))
)
# Cross-reference undeclared files, called exactly as the builder calls
# it: its declarations, not the enriched ones (a core file the platform
# also declares under another name is packed at the core's own path, 21
@@ -1283,7 +1344,7 @@ def verify_platform(
placed_by: dict[str, tuple[str, str, str]] = {}
placed = _mark_hidden(
config, undeclared, emulators_dir, db, emu_profiles, shipped, target_cores,
config, undeclared, emulators_dir, db, emu_profiles, occupied, target_cores,
region_drops, report_for_builder,
ExtraResolver(
db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index,
@@ -1363,10 +1424,23 @@ def verify_platform(
)
total = len(file_status)
# The paths a pack built from this report holds, as written in the ZIP
# (base_destination stripped, as the builder flattens it). The pipeline
# compares this set with the archive: equal counts can hide one file
# credited here and another written there.
from packpaths import _flat
pack_destinations = sorted(
{_flat(dest, base_dest, True) for dest in shipped} | {
_flat(dest, base_dest, True) for dest in placed
}
)
return {
"platform": platform,
"verification_mode": mode,
"total_files": total,
"pack_destinations": pack_destinations,
"severity_counts": counts,
"status_counts": status_counts,
"excluded_files": excluded,
@@ -1590,8 +1664,11 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
core_missing_req = len(req_not_in_repo) + len(req_hle_not_in_repo)
core_missing_opt = len(opt_not_in_repo)
# Required gaps are always named below, the optional ones on request.
listed = "" if verbose or not core_missing_opt else " (--verbose lists them)"
print(
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, {core_missing_opt} optional missing"
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, "
f"{core_missing_opt} optional missing{listed}"
)
if bundled:
names = ", ".join(sorted({u["name"] for u in bundled}))
@@ -1603,6 +1680,9 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
_print_undeclared_entry(u, "MISSING (required)", verbose)
for u in req_hle_not_in_repo:
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose)
if verbose:
for u in opt_not_in_repo:
_print_undeclared_entry(u, "MISSING (optional)", verbose)
if game_data:
gd_missing = [u for u in game_data if not u["in_repo"]]
@@ -1612,7 +1692,11 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
)
gd_unplaced = [u for u in game_data if u["in_repo"] and not u.get("in_pack", True)]
if gd_missing or gd_packed or gd_unplaced:
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing")
listed = "" if verbose or not gd_missing else " (--verbose lists them)"
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}")
if verbose:
for u in gd_missing:
_print_undeclared_entry(u, "MISSING (game data)", verbose)
if gd_unplaced:
print(
f" Game data held but not packed: {len(gd_unplaced)} "
@@ -1951,7 +2035,7 @@ def verify_emulator(
dest_hint = file_entry.get("path", "")
local_path, resolve_status = resolve_local_file(
{**file_entry, "source_profile": emu_name},
{**with_archive_members(file_entry, files), "source_profile": emu_name},
db,
zip_contents,
dest_hint=dest_hint,
@@ -2151,6 +2235,11 @@ def verify_emulator(
"emulators": [n for n, _ in selected],
"verification_mode": label,
"total_files": total,
# The paths the emulator pack fills with the profile's files, for the
# pipeline's comparison with the pack built in memory.
"pack_destinations": sorted(
dest for dest, status in file_status.items() if status == Status.OK
),
"severity_counts": counts,
"status_counts": status_counts,
"details": details,
@@ -2331,6 +2420,7 @@ def _refuse_listing_narrowings(
("--json", args.json),
("--strict", args.strict),
("--verbose", args.verbose),
("--destinations", args.destinations),
):
if on:
parser.error(f"{flag} is incompatible with {listing}")
@@ -2346,6 +2436,10 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
"--target requires --platform or --all"),
(args.target and (args.emulator or args.system),
"--target is incompatible with --emulator and --system"),
(args.destinations and not (args.platform or args.all),
"--destinations requires --platform or --all"),
(args.destinations and getattr(args, "region", None),
"--destinations describes the full pack and is incompatible with --region"),
):
if refused:
parser.error(message)
@@ -2423,6 +2517,11 @@ def _main(held: contextlib.ExitStack) -> None:
help="Show emulator ground truth details",
)
parser.add_argument("--json", action="store_true", help="JSON output")
parser.add_argument(
"--destinations", metavar="FILE",
help="With --platform or --all, also write the paths each pack holds "
"according to this report to FILE (JSON), for comparison with the archive",
)
parser.add_argument(
"--strict",
action="store_true",
@@ -2555,6 +2654,15 @@ def _main(held: contextlib.ExitStack) -> None:
if len(group_results) > 1:
print(_overall_summary(group_results))
if args.destinations:
held = {}
for (result, names), (group, _rep) in zip(group_results, groups):
for name, key in zip(names, group):
held[name] = {
"platform": key, "pack_destinations": result["pack_destinations"],
}
Path(args.destinations).write_text(json.dumps(held, indent=1, sort_keys=True))
if args.json:
for r in all_results.values():
r["details"] = _json_details(r["details"], args.verbose)
+60
View File
@@ -349,5 +349,65 @@ class AnArchiveRankedByWhatItHolds(unittest.TestCase):
self.assertIn('entry["contents"] = f["contents"]', report)
class ALooseEntryNamingItsArchive(unittest.TestCase):
"""gngeopi declares neogeo.zip loose and, apart, the ROMs it opens inside.
The archive pass took the copy holding them, the loose entry resolved by
name alone to another copy, and the profile blocked its own archive:
verify said missing, the pack said missing, and the ZIP held it."""
def test_verify_and_the_pack_ship_the_archive_once(self):
import io
from contextlib import redirect_stdout
import yaml
from common import _emulator_profiles_cache
from generate_pack import generate_emulator_pack
from verify import verify_emulator
_emulator_profiles_cache.clear()
(REPO_ROOT / "tmp").mkdir(exist_ok=True)
root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
self.addCleanup(shutil.rmtree, root, True)
previous = os.getcwd()
os.chdir(root)
self.addCleanup(os.chdir, previous)
roms = {"bios.rom": b"B" * 32, "lo.lo": b"L" * 32, "rev.rom": b"R" * 32}
files, by_name = {}, {}
# Two revisions, neither holding the other: index order picks A.
for path, members in (("bios/A/set.zip", ["bios.rom", "rev.rom"]),
("bios/B/set.zip", ["bios.rom", "lo.lo"])):
Path(path).parent.mkdir(parents=True)
with zipfile.ZipFile(path, "w") as zf:
for member in members:
zf.writestr(member, roms[member])
sha1 = hashlib.sha1(Path(path).read_bytes()).hexdigest()
files[sha1] = {"path": path, "name": "set.zip", "sha1": sha1, "size": 1}
by_name.setdefault("set.zip", []).append(sha1)
db = {"files": files, "indexes": {"by_name": by_name, "by_md5": {},
"by_path_suffix": {}, "by_crc32": {}}}
Path("emulators").mkdir()
Path("emulators/neo.yml").write_text(yaml.safe_dump({
"emulator": "Neo", "type": "libretro", "systems": ["snk-neogeo-mvs"],
"files": [{"name": "set.zip", "required": True, "category": "bios_zip"}] + [
{"name": name, "archive": "set.zip", "required": True,
"crc32": format(zlib.crc32(data), "08x")}
for name, data in roms.items() if name != "rev.rom"
],
}))
report = verify_emulator(["neo"], "emulators", db, platforms_dir="platforms")
with redirect_stdout(io.StringIO()) as out:
zip_path = generate_emulator_pack(
["neo"], "emulators", db, "bios", "out", offline=True,
platforms_dir="platforms",
)
with zipfile.ZipFile(zip_path) as zf, zf.open("set.zip") as inner:
held = zipfile.ZipFile(io.BytesIO(inner.read())).namelist()
self.assertEqual(sorted(held), ["bios.rom", "lo.lo"])
self.assertNotIn("missing", out.getvalue())
self.assertEqual({d["status"] for d in report["details"]}, {"ok"})
self.assertEqual(report["total_files"], 1)
if __name__ == "__main__":
unittest.main()
+39
View File
@@ -21,6 +21,7 @@ from cross_reference import ( # noqa: E402
_build_supplemental_index,
cross_reference,
entry_source,
print_report,
)
@@ -38,6 +39,44 @@ class GapReportSources(unittest.TestCase):
entry = {"name": "wave1.bin", "sha256": "AB" * 32}
self.assertEqual(entry_source(entry, _index(by_sha256={"ab" * 32: "s"})), "bios")
def test_a_name_never_wins_over_a_declared_hash(self):
"""zquestclassic pins ending.mid by sha1, the collection holds only
Hydra Castle Labyrinth's ending.mid: the report called it held."""
other = "9" * 40
index = _index(
by_name={"ending.mid": [other]}, by_name_lower={"ending.mid": "ending.mid"},
by_path_suffix={"data/midi/ending.mid": [other]},
db_files={other: {"path": "bios/hcl/ending.mid", "md5": "1" * 32, "size": 9}},
)
pinned = {"name": "ending.mid", "sha1": "5" * 40}
self.assertIsNone(entry_source(pinned, index))
self.assertIsNone(entry_source({**pinned, "path": "data/midi/ending.mid"}, index))
self.assertEqual(entry_source({"name": "ending.mid"}, index), "bios")
self.assertEqual(entry_source({"name": "ending.mid", "sha1": other}, index), "bios")
self.assertEqual(entry_source({"name": "ending.mid", "md5": "1" * 32}, index), "bios")
def test_the_heading_and_the_total_count_bundled_files_alike(self):
"""abuse printed "Gaps: 3 missing" above three [BUNDLED] files and
"0 missing (need to be sourced)" below."""
import contextlib
import io
report = {"abuse": {
"emulator": "Abuse", "systems": [], "gaps": 3, "total_files": 3,
"platform_covered": 0, "gap_missing": 3,
"gap_details": [
{"name": f"f{i}", "required": True, "source": "missing", "bundled": True,
"note": ""}
for i in range(3)
],
}}
with contextlib.redirect_stdout(io.StringIO()) as out:
print_report(report)
text = out.getvalue()
self.assertIn("Gaps: 3 bundled", text)
self.assertNotIn("3 missing", text)
self.assertIn("0 missing (need to be sourced)", text)
def test_a_directory_does_not_stand_for_a_file(self):
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
(Path(tmp) / "bios" / "cpcemu" / "ROM").mkdir(parents=True)
+34
View File
@@ -83,6 +83,40 @@ class ReadersResolve(unittest.TestCase):
self.assertNotIn("ioquake3", et)
class ReadersFollowTheDataTrees(unittest.TestCase):
"""Dolphin names codehandler.bin without a hash. The pack takes it from
data/dolphin-sys, and the site credited Dolphin and Triforce with the
Dolphin-MMJR copy, another revision, because the consumers resolved
without the data trees. Reads the repository."""
def test_dolphin_does_not_read_the_mmjr_copy(self):
import os
from common import load_database, load_emulator_profiles
if not (REPO_ROOT / "database.json").is_file() or not (
REPO_ROOT / "data" / "dolphin-sys" / "codehandler.bin"
).is_file():
self.skipTest("database.json or data/dolphin-sys is absent")
previous = os.getcwd()
os.chdir(REPO_ROOT)
try:
db = load_database("database.json")
profiles = load_emulator_profiles("emulators")
consumers = FileConsumers(db)
consumers.add_emulators({"dolphin": profiles["dolphin"]})
finally:
os.chdir(previous)
mmjr = next(
(sha1 for sha1, entry in db["files"].items()
if entry["path"] == "bios/Other/dolphin-mmjr/codehandler.bin"),
None,
)
if mmjr is None:
self.skipTest("the Dolphin-MMJR copy is not collected")
self.assertNotIn("dolphin", consumers.emulators.get(mmjr, set()))
def tearDownModule():
_TMP.cleanup()
+28 -1
View File
@@ -15,7 +15,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
from cross_reference import cross_reference # noqa: E402
from validation import declaration_settles # noqa: E402
from validation import case_folded_slots, declaration_settles # noqa: E402
SC55 = {"name": "ROM1.BIN", "path": "SC-55/ROM1.BIN", "size": 32768,
"validation": ["size"], "required": True}
@@ -45,6 +45,33 @@ class OneRule(unittest.TestCase):
self.assertIn("declaration_settles(", source, module)
class ACaseFoldingFilesystem(unittest.TestCase):
"""RetroBat declares 32X_G_BIOS.BIN, BlastEm opens 32X_G_BIOS.bin at the
same place. The exact-case name test reported it undeclared before the
slot was ever compared."""
declared = {"32X_G_BIOS.BIN": [(256, "32x_g_bios.bin")]}
entry = {"name": "32X_G_BIOS.bin"}
def test_the_slot_settles_it_where_case_folds(self):
folded = case_folded_slots({"case_insensitive_fs": True}, self.declared)
self.assertTrue(
declaration_settles(self.entry, "32X_G_BIOS.bin", self.declared, "", folded)
)
def test_two_spellings_are_two_files_elsewhere(self):
folded = case_folded_slots({"case_insensitive_fs": False}, self.declared)
self.assertFalse(
declaration_settles(self.entry, "32X_G_BIOS.bin", self.declared, "", folded)
)
def test_another_place_settles_nothing(self):
folded = case_folded_slots({"case_insensitive_fs": True}, self.declared)
self.assertFalse(declaration_settles(
self.entry, "32X_G_BIOS.bin", self.declared, "blastem/32X_G_BIOS.bin", folded
))
class TheRunningBuildPlacesTheGap(unittest.TestCase):
"""Batocera runs Cemu standalone, which reads keys.txt at its
standalone_path. The gap report gave the libretro Cemu/keys.txt while
+22 -10
View File
@@ -98,7 +98,7 @@ class TheReportSaysSo(unittest.TestCase):
name: verify_platform(
load_platform_config(name, "platforms"), db, "emulators", profiles
)
for name in ("retroarch", "retrobat")
for name in ("retroarch", "retrobat", "lakka")
}
finally:
os.chdir(previous)
@@ -109,16 +109,28 @@ class TheReportSaysSo(unittest.TestCase):
return u
self.skipTest(f"{emulator} {path} is not in the cross-reference")
def test_a_case_twin_with_other_bytes_is_left_out(self):
entry = self._entry("NP2kai", "np2kai/bios.rom", "retrobat")
self.assertIs(entry.get("in_pack"), False)
self.assertIn("another case", entry.get("omitted", ""))
def _listed(self, emulator: str, path: str, platform: str) -> bool:
return any(
u.get("emulator") == emulator and (u.get("path") or u.get("name")) == path
for u in self.results[platform]["undeclared_files"]
)
def test_a_twin_with_the_same_bytes_serves_the_core(self):
# MesenCE opens bios.col, and RetroArch's pack folds it onto BIOS.col,
# the same bytes, which serves the core on a folding filesystem.
entry = self._entry("MesenCE", "bios.col")
self.assertIsNot(entry.get("in_pack"), False)
def test_a_declaration_in_another_case_settles_the_slot(self):
"""On a filesystem that folds case the platform's file at the slot is
the one the core opens, whatever its spelling, and the slot
arbitration judges its bytes. Listed as a core file, RetroBat's
32X_G_BIOS.BIN counted BlastEm's 32X_G_BIOS.bin in the pack twice."""
for platform, emulator, path in (
("retrobat", "NP2kai", "np2kai/bios.rom"),
("retrobat", "BlastEm", "32X_G_BIOS.bin"),
("retroarch", "MesenCE", "bios.col"),
):
with self.subTest(platform=platform, path=path):
self.assertFalse(self._listed(emulator, path, platform))
def test_a_filesystem_keeping_both_names_reports_the_other(self):
"""Lakka runs on Linux, where bios.col and BIOS.col are two files."""
self.assertTrue(self._listed("MesenCE", "bios.col", "lakka"))
def test_a_directory_under_a_packed_file_is_left_out(self):
entry = self._entry("nSide (SFC Balanced)", "SGB1.sfc/lr35902.boot.rom")
+195
View File
@@ -0,0 +1,195 @@
"""verify and the pack name the same paths, not only the same number of them.
The pipeline compared counts. RetroArch declares SGB1.sfc and, below it,
SGB1.sfc/program.rom: the pack writes the file and skips the path it blocks,
verify credited both, and both sides still read 526/526. A user_provided
declaration keeps its slot in the pack, while verify let a core file take
it and counted one more file from the cores.
"""
from __future__ import annotations
import contextlib
import hashlib
import io
import json
import os
import shutil
import sys
import tempfile
import unittest
import zipfile
from pathlib import Path
import yaml
REPO_ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(REPO_ROOT / "scripts"))
class _Fixture(unittest.TestCase):
def setUp(self) -> None:
from common import _emulator_profiles_cache
_emulator_profiles_cache.clear()
(REPO_ROOT / "tmp").mkdir(exist_ok=True)
self.root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
self.addCleanup(shutil.rmtree, self.root, True)
self.bios = self.root / "bios"
self.platforms = self.root / "platforms"
self.emulators = self.root / "emulators"
for directory in (self.bios, self.platforms, self.emulators):
directory.mkdir()
files, by_name, by_md5 = {}, {}, {}
for name, data in (("sgb.sfc", b"SGB" * 8), ("program.rom", b"PRG" * 8),
("core.bin", b"CORE" * 8)):
path = self.bios / name
path.write_bytes(data)
sha1, md5 = hashlib.sha1(data).hexdigest(), hashlib.md5(data).hexdigest()
files[sha1] = {"path": str(path), "name": name, "md5": md5, "size": len(data)}
by_name[name] = [sha1]
by_md5[md5] = sha1
self.db = {"files": files, "indexes": {"by_name": by_name, "by_md5": by_md5,
"by_crc32": {}, "by_path_suffix": {}}}
def _platform(self, files: list[dict], cores: list[str] | None = None) -> dict:
config = {"platform": "Paths", "verification_mode": "existence",
"base_destination": "system", "systems": {"sys": {"files": files}}}
if cores is not None:
config["cores"] = cores
(self.platforms / "paths.yml").write_text(yaml.safe_dump(config))
return config
def _both(self) -> tuple[dict, set[str]]:
from common import load_emulator_profiles, load_platform_config
from generate_pack import generate_pack
from verify import verify_platform
profiles = load_emulator_profiles(str(self.emulators))
report = verify_platform(
load_platform_config("paths", str(self.platforms)), self.db,
str(self.emulators), profiles,
)
with contextlib.redirect_stdout(io.StringIO()):
zip_path = generate_pack(
"paths", str(self.platforms), self.db, str(self.bios),
str(self.root / "out"), emulators_dir=str(self.emulators),
emu_profiles=profiles, offline=True,
)
with zipfile.ZipFile(zip_path) as zf:
members = {n for n in zf.namelist() if n != "README.txt"}
return report, members
class APathBlockedByAFile(_Fixture):
def test_verify_credits_only_the_file_the_pack_writes(self):
self._platform([
{"name": "sgb.sfc", "destination": "SGB1.sfc"},
{"name": "program.rom", "destination": "SGB1.sfc/program.rom"},
])
report, members = self._both()
self.assertEqual(members, {"SGB1.sfc"})
self.assertEqual(set(report["pack_destinations"]), members)
class AUserProvidedSlot(_Fixture):
def test_no_core_file_takes_it_on_either_side(self):
(self.emulators / "coreemu.yml").write_text(yaml.safe_dump({
"emulator": "CoreEmu", "type": "libretro", "systems": ["sys"],
"files": [{"name": "core.bin", "required": True}],
}))
self._platform(
[{"name": "manual.bin", "destination": "core.bin", "storage": "user_provided"}],
cores=["coreemu"],
)
report, members = self._both()
self.assertEqual(members, {"INSTRUCTIONS_manual.bin.txt"})
self.assertEqual(report["packed_extras"], {"bios": 0, "game_data": 0})
self.assertEqual(report["pack_destinations"], [])
class AnEntryWithoutAFileTakesNoPath(unittest.TestCase):
"""EmuDeck's system:md5 entries name no file. verify credited the bare
base destination as a pack path, and the comparison failed on ''."""
def test_it_is_left_out(self):
from verify import _builder_placement
shipped, occupied = _builder_placement(
[("s", "bios/", None), ("s", "", "x"), ("s", "bios/a.bin", "bios/a.bin")], False
)
self.assertEqual(shipped, {"bios/a.bin": "bios/a.bin"})
self.assertEqual(set(occupied), {"bios/a.bin"})
class ThePipelineComparesPaths(unittest.TestCase):
"""Equal counts, other paths: the comparison fails and names both sides."""
def test_a_swap_at_equal_counts_fails(self):
import pipeline
(REPO_ROOT / "tmp").mkdir(exist_ok=True)
root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
self.addCleanup(shutil.rmtree, root, True)
zip_path = root / "Plat_BIOS_Pack.zip"
with zipfile.ZipFile(zip_path, "w") as zf:
for name in ("a.bin", "b.bin", "README.txt", "INSTRUCTIONS_x.bin.txt"):
zf.writestr(name, name)
output = f"Generating pack for plat [source=full]...\n {zip_path}: 2 files packed\n"
for credited, expected in ((["a.bin", "b.bin"], True), (["a.bin", "c.bin"], False)):
with self.subTest(credited=credited):
destinations = root / "d.json"
destinations.write_text(json.dumps(
{"Plat": {"platform": "plat", "pack_destinations": credited}}
))
previous = os.getcwd()
os.chdir(REPO_ROOT)
try:
with mock_tree_members(), contextlib.redirect_stdout(io.StringIO()) as out:
ok = pipeline.check_pack_destinations(str(destinations), output)
finally:
os.chdir(previous)
self.assertIs(ok, expected, out.getvalue())
if not expected:
self.assertIn("verify only 1: c.bin", out.getvalue())
self.assertIn("pack only 1: b.bin", out.getvalue())
def test_the_run_compares_every_emulator_pack(self):
source = (REPO_ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8")
self.assertIn("ok = check_pack_destinations(destinations_file, pack_output) and ok", source)
self.assertIn("ok = check_emulator_packs() and ok", source)
self.assertIn('"--destinations", destinations_file', source)
@contextlib.contextmanager
def mock_tree_members():
import pipeline
original = pipeline._tree_members
pipeline._tree_members = lambda *_args: {}
try:
yield
finally:
pipeline._tree_members = original
class EveryEmulatorPackMatchesItsReport(unittest.TestCase):
"""Reads the repository: every profile, both builds, built in memory."""
def test_the_paths_agree(self):
if not (REPO_ROOT / "database.json").is_file():
self.skipTest("database.json is not built")
import pipeline
previous = os.getcwd()
os.chdir(REPO_ROOT)
try:
with contextlib.redirect_stdout(io.StringIO()) as out:
ok = pipeline.check_emulator_packs()
finally:
os.chdir(previous)
self.assertTrue(ok, out.getvalue())
if __name__ == "__main__":
unittest.main()
+58
View File
@@ -1223,5 +1223,63 @@ class NoMarketStopsAtAustralia(unittest.TestCase):
self.assertEqual(tagged, [])
class ARefusedRegionalFileHoldsNoSlot(unittest.TestCase):
"""rg_us.bin is held but its md5 contradicts the one the core checks, so
the emulator pack refuses it. Counted held, it won the region and the
Japanese BIOS was withdrawn: --emulator --region us shipped no BIOS for
the system, and verify, reading the same drops, did not see it."""
def test_the_fallback_stays_in_pack_and_report(self):
import hashlib
import shutil
import tempfile
import zipfile
import yaml
from generate_pack import generate_emulator_pack
from verify import verify_emulator
common._emulator_profiles_cache.clear()
repo = Path(__file__).resolve().parents[1]
(repo / "tmp").mkdir(exist_ok=True)
root = Path(tempfile.mkdtemp(dir=repo / "tmp"))
self.addCleanup(shutil.rmtree, root, True)
previous = os.getcwd()
os.chdir(root)
self.addCleanup(os.chdir, previous)
files, by_name = {}, {}
for name, data in (("rg_us.bin", b"NOT THE US DUMP"), ("rg_jp.bin", b"JP DUMP")):
path = Path("bios") / name
path.parent.mkdir(exist_ok=True)
path.write_bytes(data)
sha1 = hashlib.sha1(data).hexdigest()
files[sha1] = {"path": str(path), "name": name, "size": len(data),
"md5": hashlib.md5(data).hexdigest()}
by_name[name] = [sha1]
db = {"files": files, "indexes": {"by_name": by_name, "by_md5": {},
"by_crc32": {}, "by_path_suffix": {}}}
Path("emulators").mkdir()
Path("emulators/rgemu.yml").write_text(yaml.safe_dump({
"emulator": "RgEmu", "type": "libretro", "systems": ["sys"],
"files": [
{"name": "rg_us.bin", "region": ["north-america"], "required": True,
"md5": hashlib.md5(b"THE US DUMP").hexdigest(), "validation": ["md5"]},
{"name": "rg_jp.bin", "region": ["japan"], "required": True},
],
}))
with contextlib.redirect_stdout(io.StringIO()):
zip_path = generate_emulator_pack(
["rgemu"], "emulators", db, "bios", "out", regions=["north-america"],
offline=True, platforms_dir="platforms",
)
with zipfile.ZipFile(zip_path) as zf:
self.assertIn("rg_jp.bin", zf.namelist())
report = verify_emulator(
["rgemu"], "emulators", db, regions=["north-america"], platforms_dir="platforms",
)
self.assertIn("rg_jp.bin", report["pack_destinations"])
if __name__ == "__main__":
unittest.main()
+63
View File
@@ -349,6 +349,7 @@ class TestArbitration(unittest.TestCase):
def _conflict(self):
platform = slots.Claim(
"platform", "GC/JAP/IPL.bin", "IPL.bin",
entry={"md5": "m" * 32, "sha1": "a" * 40},
local_path="bios/Console/GC/USA/IPL.bin", status="md5_exact",
)
profile = slots.Claim(
@@ -376,13 +377,75 @@ class TestArbitration(unittest.TestCase):
self.assertEqual(decision.winner.origin, "profile")
self.assertFalse(decision.serves_both)
def test_a_hashless_entry_is_a_path_in_an_md5_platform(self):
"""EmuDeck declares neocdz.zip with no md5 and checkBIOS.sh never
names it: the frontend checks nothing there, so the core's file
serves both."""
conflict = self._conflict()
conflict.platform_claim.entry = {"name": "IPL.bin"}
for mode in ("md5", "sha1"):
with self.subTest(mode=mode):
self.assertTrue(slots.arbitrate(conflict, mode).serves_both)
# RomM compares size then any hash: a hash of another kind is content.
conflict.platform_claim.entry = {"name": "IPL.bin", "sha1": "a" * 40, "size": 3}
self.assertFalse(
slots.arbitrate(conflict, "md5", rule="size_and_any_hash").serves_both
)
conflict.platform_claim.entry = {"name": "x.zip", "zipped_file": "IPL.bin"}
self.assertFalse(slots.arbitrate(conflict, "md5").serves_both)
def test_the_pack_serves_the_proven_file_at_a_hashless_md5_entry(self):
config = {
"verification_mode": "md5",
"cores": ["dolphin"],
"systems": {"console": {"files": [
{"name": "IPL.bin", "destination": "IPL.bin"},
{"name": "IPL.bin", "destination": "hashed/IPL.bin", "md5": "m" * 32},
]}},
}
profiles = {"dolphin": {"type": "libretro", "systems": ["console"], "files": [
{"name": "IPL.bin", "md5": "n" * 32},
{"name": "IPL.bin", "path": "hashed/IPL.bin", "md5": "n" * 32},
]}}
overrides = slots.pack_overrides(config, profiles, REGIONS_DB)
# The hashed declaration keeps the file the frontend checks.
self.assertEqual(overrides, {"IPL.bin": "bios/Console/GC/JAP/IPL.bin"})
def test_the_reported_line_gives_the_ground_for_the_decision(self):
kept = slots.format_decision(slots.arbitrate(self._conflict(), "md5"))
self.assertIn("verifies content", kept)
# A fact about each layer, no verdict on the platform's list.
self.assertIn("the file dolphin reads", kept)
self.assertNotIn("wrong", slots.__loader__.get_source("slots").lower())
served = slots.format_decision(slots.arbitrate(self._conflict(), "existence"))
self.assertIn("both are satisfied", served)
class TheStrongestProofTakesAnUndeclaredPath(unittest.TestCase):
"""No platform declares RetroDECK's hiscore.dat. fbalpha2012 asks for it
by name, mame2015 pins it by sha1: the first in profile order took the
path, whatever the proof, and mame2015 received another revision."""
def test_a_held_hash_beats_a_bare_name_whatever_the_order(self):
from packextras import _collect_emulator_extras
config = {
"platform": "P", "verification_mode": "existence",
"cores": ["aaa", "zzz"],
"systems": {"console": {"files": [{"name": "other.bin", "destination": "other.bin"}]}},
}
profiles = {
"aaa": {"emulator": "Aaa", "type": "libretro", "systems": ["console"],
"files": [{"name": "IPL.bin", "required": True}]},
"zzz": {"emulator": "Zzz", "type": "libretro", "systems": ["console"],
"files": [{"name": "IPL.bin", "required": True, "md5": "n" * 32}]},
}
extras = _collect_emulator_extras(config, "", REGIONS_DB, set(), "", profiles)
winner = next(e for e in extras if e["destination"] == "IPL.bin")
self.assertEqual(winner["source_profile"], "zzz")
self.assertEqual({c["source_profile"] for c in winner["other_claims"]}, {"aaa"})
class TestBuilderAndVerifierAgree(unittest.TestCase):
"""The two tools must settle a contested destination the same way."""
+38
View File
@@ -148,5 +148,43 @@ class ExistenceReportsWhatThePackReports(unittest.TestCase):
self.assertEqual(result["status"], verify.Status.OK)
self.assertIn("declared hash 9d2ff233", result["discrepancy"])
class TheDestinationsFlagNeedsAPack(unittest.TestCase):
"""--destinations describes a platform's full pack: another mode refuses it."""
def test_refused_outside_platform_modes(self):
for mode in (("--emulator", "handy"), ("--platform", "retroarch", "--region", "us")):
with self.subTest(mode=mode):
result = _run("scripts/verify.py", *mode, "--destinations", "tmp/never.json")
self.assertEqual(result.returncode, 2, result.stdout[-300:])
self.assertIn("--destinations", result.stderr)
class EveryGapCountedIsNamed(unittest.TestCase):
"""verify --platform batocera --verbose counted 3 optional core files and
427 game data files missing and named none of them."""
def test_verbose_names_them_and_the_summary_says_how(self):
import contextlib
import io
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import verify # noqa: PLC0415
result = {"undeclared_files": [
{"name": "opt.rom", "emulator": "core", "required": False, "in_repo": False},
{"name": "level.dat", "emulator": "engine", "required": True, "in_repo": False,
"category": "game_data"},
]}
with contextlib.redirect_stdout(io.StringIO()) as quiet:
verify._print_undeclared_section(result, verbose=False)
self.assertIn("1 optional missing (--verbose lists them)", quiet.getvalue())
self.assertIn("1 missing (--verbose lists them)", quiet.getvalue())
with contextlib.redirect_stdout(io.StringIO()) as verbose:
verify._print_undeclared_section(result, verbose=True)
self.assertIn("MISSING (optional): core needs opt.rom", verbose.getvalue())
self.assertIn("MISSING (game data): engine needs level.dat", verbose.getvalue())
if __name__ == "__main__":
unittest.main()
+2 -2
View File
@@ -510,7 +510,7 @@ This executes in sequence:
2. `refresh_data_dirs.py` - update data directories (skipped with `--offline`)
3. `verify.py --all` - verify all platforms including the new one
4. `generate_pack.py --all` - build ZIP packs + install and target manifests
5. Consistency check - verify counts match between verify and pack
5. Consistency check: verify and the pack agree on counts and on the paths each archive holds
6. Pack integrity - extract ZIPs and verify hashes per platform mode
7. `generate_readme.py` - regenerate README and CONTRIBUTING
8. `generate_site.py` - regenerate documentation site and `mkdocs.yml`
@@ -527,7 +527,7 @@ Check the output for:
- Pack generation succeeds and includes the expected files
- Consistency check passes (verify file counts match pack file counts)
Verification is not optional. A platform that passes `pipeline.py` today may break tomorrow if upstream changes its data format. Run the full pipeline on every change, even if the modification seems trivial. The consistency check (verify counts must match pack counts) catches subtle issues where files resolve during verification but fail during pack generation, or vice versa.
Verification is not optional. A platform that passes `pipeline.py` today may break tomorrow if upstream changes its data format. Run the full pipeline on every change, even if the modification seems trivial. The consistency check (verify and the pack must agree on counts and paths) catches subtle issues where files resolve during verification but fail during pack generation, or vice versa.
## Checklist
+1 -1
View File
@@ -38,7 +38,7 @@ Every module in `tests/`, by the area it guards.
| Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` |
| File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` |
| Verification, cross-reference and profile checks | synthetic + profiles | `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_cross_reference_sources`, `test_file_consumers`, `test_fileless_audit`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice` |
| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_pack_conformance_dirs`, `test_pack_counts`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_pack_conformance_dirs`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
| Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
| Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` |
+8 -4
View File
@@ -42,7 +42,7 @@ Pipeline steps:
| 4b | Generate install manifests | `--skip-packs` |
| 4c | Generate target manifests | `--skip-packs` |
| 4d | Collected copies only an emulator install provides (`bundled_only.py --strict`, gates the run) | - |
| 5 | Consistency check | if verify or pack skipped |
| 5 | Consistency check: counts, paths per archive, every emulator pack | if verify or pack skipped |
| 6 | Pack integrity (extract + hash) | `--skip-packs` |
| 7 | Generate README | `--skip-docs` |
| 8 | Generate site | `--skip-docs` |
@@ -78,9 +78,12 @@ alone, run `python scripts/check_buildbot_system.py`.
and manifest names. `--manifest --target switch` once overwrote the
manifest every installer downloads with a narrowed one.
- **A step that did not run says SKIPPED**, never OK.
- **verify.py and generate_pack.py read the same functions.** A count that
differs between the report and the pack is a defect, which is what step 5
checks.
- **verify.py and generate_pack.py read the same functions.** A count or a
path that differs between the report and the pack is a defect, which is
what step 5 checks: the counts, the paths each archive holds against the
ones verify credits (`--destinations`), and every emulator pack, each
profile in each build, built in memory against `verify --emulator`.
Equal counts once hid four RetroArch paths credited and never written.
- **Generators write through `write_if_changed()`.** A new timestamp alone
rewrites nothing, so a diff on a generated file always means content.
- **One run per output directory.** Pack generation takes a lock on
@@ -118,6 +121,7 @@ python scripts/verify.py --emulator handy --strict # exit 1 on a required file
python scripts/verify.py --system atari-lynx # single system
python scripts/verify.py --platform retroarch --target switch # filter by hardware
python scripts/verify.py --platform recalbox --region us # regional priority list
python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds
python scripts/verify.py --list-emulators # list all emulators
python scripts/verify.py --list-systems # list all systems
python scripts/verify.py --platform retroarch --list-targets # list available targets