mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-23 20:29:06 -05:00
chore: merge audit fixes (integrity)
# Conflicts: # install/batocera.json # install/emudeck.json # install/lakka.json # install/recalbox.json # install/retroarch.json # install/retrobat.json # install/retrodeck.json # install/retropie.json # install/rocknix.json # install/romm.json # scripts/generate_pack.py # wiki/tools.md
This commit is contained in:
commit
378a9e4da5
19 files changed
+1199
-101
No files matched your search
+88
-22
@@ -31,6 +31,7 @@ from common import (
|
||||
get_mame_clone_map,
|
||||
list_emulator_profiles,
|
||||
list_registered_platforms,
|
||||
load_data_dir_registry,
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
@@ -44,7 +45,7 @@ from common import (
|
||||
select_emulator_profiles,
|
||||
)
|
||||
from hashing import parse_hash_list
|
||||
from validation import declaration_settles, outside_gap_scope
|
||||
from validation import case_folded_slots, declaration_settles, outside_gap_scope
|
||||
|
||||
yaml = require_yaml()
|
||||
|
||||
@@ -105,6 +106,25 @@ def declared_names(
|
||||
)
|
||||
|
||||
|
||||
def folded_declared_slots(
|
||||
platforms_dir: str, platforms: list[str] | None, db: dict
|
||||
) -> frozenset[str]:
|
||||
"""The slots platforms whose filesystem folds case fill, any spelling.
|
||||
|
||||
RetroBat's 32X_G_BIOS.BIN is the file BlastEm opens as 32X_G_BIOS.bin,
|
||||
and the exact-case name test called it undeclared.
|
||||
"""
|
||||
from common import platform_declarations
|
||||
|
||||
slots: set[str] = set()
|
||||
for platform_name in platforms or list_registered_platforms(
|
||||
platforms_dir, include_archived=True
|
||||
):
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
slots |= case_folded_slots(config, platform_declarations(config, db))
|
||||
return frozenset(slots)
|
||||
|
||||
|
||||
def _build_supplemental_index(
|
||||
data_root: str = "data", bios_root: str = "bios"
|
||||
) -> set[str]:
|
||||
@@ -178,21 +198,28 @@ def _resolve_source(
|
||||
or ``None`` (not available anywhere).
|
||||
|
||||
A name hit whose size contradicts a size the emulator verifies is not the
|
||||
file, so it does not count as available.
|
||||
file, so it does not count as available. Nor is one whose bytes carry
|
||||
none of the hashes the entry declares: a name never wins over a declared
|
||||
hash, and zquestclassic's ending.mid read as held because Hydra Castle
|
||||
Labyrinth has another file of that name.
|
||||
"""
|
||||
declared = _declared_content(file_entry or {})
|
||||
|
||||
def _name_hit(name: str) -> bool:
|
||||
"""Whether the files indexed under *name* can be this entry."""
|
||||
def _can_be(sha1s: list | str) -> bool:
|
||||
if not file_entry or db_files is None:
|
||||
return True
|
||||
sha1s = by_name.get(name) or []
|
||||
if not isinstance(sha1s, list):
|
||||
sha1s = [sha1s]
|
||||
return any(
|
||||
name_match_size_ok(file_entry, db_files.get(sha1, {}).get("size"))
|
||||
and _record_carries(sha1, db_files.get(sha1, {}), declared)
|
||||
for sha1 in sha1s
|
||||
)
|
||||
|
||||
def _name_hit(name: str) -> bool:
|
||||
"""Whether the files indexed under *name* can be this entry."""
|
||||
return _can_be(by_name.get(name) or [])
|
||||
|
||||
# bios/ via database.json by_name
|
||||
if fname in by_name and _name_hit(fname):
|
||||
return "bios"
|
||||
@@ -209,7 +236,7 @@ def _resolve_source(
|
||||
):
|
||||
return "bios"
|
||||
# bios/ via by_path_suffix (regional variants)
|
||||
if by_path_suffix and fname in by_path_suffix:
|
||||
if by_path_suffix and fname in by_path_suffix and _can_be(by_path_suffix[fname]):
|
||||
return "bios"
|
||||
# bios/ under the canonical MAME set name, as resolve_local_file does:
|
||||
# a renamed archive is held once, under the name the dedup kept.
|
||||
@@ -252,6 +279,23 @@ def _data_hit(
|
||||
_HASH_FIELDS = ("sha1", "md5", "sha256", "crc32")
|
||||
|
||||
|
||||
def _record_carries(sha1: str, record: dict, declared: dict[str, set[str]]) -> bool:
|
||||
"""Whether a database record has one of the declared hashes, or none is declared."""
|
||||
if not declared:
|
||||
return True
|
||||
actual = {
|
||||
"sha1": sha1.lower(),
|
||||
**{field: str(record.get(field) or "").lower() for field in ("md5", "sha256", "crc32")},
|
||||
}
|
||||
return any(
|
||||
actual[field] and (
|
||||
actual[field] in values
|
||||
or (field == "md5" and any(len(v) < 32 and actual[field].startswith(v) for v in values))
|
||||
)
|
||||
for field, values in declared.items()
|
||||
)
|
||||
|
||||
|
||||
def _declared_content(file_entry: dict) -> dict[str, set[str]]:
|
||||
"""The hashes an entry declares, lowercased, by algorithm."""
|
||||
declared: dict[str, set[str]] = {}
|
||||
@@ -326,9 +370,21 @@ class FileConsumers:
|
||||
file by the evidence the pack uses, so a homonym is nobody's file.
|
||||
"""
|
||||
|
||||
def __init__(self, db: dict, zip_contents: dict | None = None) -> None:
|
||||
def __init__(
|
||||
self, db: dict, zip_contents: dict | None = None,
|
||||
data_dir_registry: dict | None = None,
|
||||
) -> None:
|
||||
self._db = db
|
||||
self._zip_contents = zip_contents
|
||||
# The builder resolves with the data trees: Dolphin's codehandler.bin
|
||||
# comes from data/dolphin-sys, and without them the name step gave it
|
||||
# Dolphin-MMJR's copy, which the site then said Dolphin reads.
|
||||
self._data_dir_registry = (
|
||||
data_dir_registry if data_dir_registry is not None
|
||||
else load_data_dir_registry(
|
||||
str(Path(__file__).resolve().parent.parent / DEFAULT_PLATFORMS_DIR)
|
||||
)
|
||||
)
|
||||
self._sha1_by_path = {
|
||||
path: sha1
|
||||
for sha1, entry in db.get("files", {}).items()
|
||||
@@ -362,7 +418,8 @@ class FileConsumers:
|
||||
if owner:
|
||||
entry = {**entry, "source_profile": owner}
|
||||
path, status = resolve_local_file(
|
||||
entry, self._db, self._zip_contents, dest_hint=dest
|
||||
entry, self._db, self._zip_contents, dest_hint=dest,
|
||||
data_dir_registry=self._data_dir_registry,
|
||||
)
|
||||
if not path or status in ("not_found", "hash_mismatch"):
|
||||
return None
|
||||
@@ -470,6 +527,7 @@ def entry_source(f: dict, index: dict) -> str | None:
|
||||
{"indexes": {"by_path_suffix": by_path_suffix}}, path_field or fname
|
||||
):
|
||||
return "bios"
|
||||
found = None
|
||||
for candidate in [fname, *([path_field] if path_field != fname else []),
|
||||
*(f.get("aliases") or [])]:
|
||||
if not candidate:
|
||||
@@ -478,12 +536,15 @@ def entry_source(f: dict, index: dict) -> str | None:
|
||||
candidate, by_name, by_name_lower, data_names, by_path_suffix, f, db_files,
|
||||
index.get("bios_dir"),
|
||||
)
|
||||
if source is not None:
|
||||
if source == "bios":
|
||||
return source
|
||||
found = found or source
|
||||
|
||||
def values(field: str) -> list[str]:
|
||||
return parse_hash_list(f.get(field))
|
||||
|
||||
# The declared content anywhere in bios/ comes before a data/ copy, as in
|
||||
# the resolver: the hash steps run before the data directories.
|
||||
held = (
|
||||
any(index["by_md5"].get(v) for v in parse_md5_list(f.get("md5")))
|
||||
or any(v in db_files for v in values("sha1"))
|
||||
@@ -491,7 +552,7 @@ def entry_source(f: dict, index: dict) -> str | None:
|
||||
or any(v in index["by_sha256"] for v in values("sha256"))
|
||||
or any(index["by_crc32"].get(v) for v in values("crc32"))
|
||||
)
|
||||
return "bios" if held else None
|
||||
return "bios" if held else found
|
||||
|
||||
|
||||
def _resolve_archive_source(
|
||||
@@ -629,7 +690,9 @@ def _cross_reference_profile(
|
||||
# Check platform declaration: by name, by an alias the core answers
|
||||
# to (RetroArch lists amiberry's cd32.rom as kick40060.CD32), or by
|
||||
# archive.
|
||||
in_platform = declaration_settles(f, fname, platform_names, effective_path)
|
||||
in_platform = declaration_settles(
|
||||
f, fname, platform_names, effective_path, index.get("folded_slots", frozenset())
|
||||
)
|
||||
|
||||
if in_platform:
|
||||
seen_files.add(seen_key)
|
||||
@@ -736,6 +799,7 @@ def cross_reference(
|
||||
data_names: set[str] | None = None,
|
||||
all_declared: dict[str, list[tuple[int | None, str]]] | None = None,
|
||||
standalone_cores: set[str] | None = None,
|
||||
folded_slots: frozenset[str] = frozenset(),
|
||||
) -> dict:
|
||||
"""Compare emulator profiles against platform declarations.
|
||||
|
||||
@@ -772,6 +836,7 @@ def cross_reference(
|
||||
"db_files": db_files,
|
||||
"data_names": data_names,
|
||||
"all_declared": all_declared,
|
||||
"folded_slots": folded_slots,
|
||||
"bios_dir": db.get("bios_dir"),
|
||||
}
|
||||
for emu_name, profile in profiles.items():
|
||||
@@ -807,11 +872,16 @@ def print_report(
|
||||
if gaps == 0 and not show_clean:
|
||||
continue
|
||||
|
||||
parts = []
|
||||
for key in ("bios", "data", "large_file", "missing"):
|
||||
count = data.get(f"gap_{key}", 0)
|
||||
if count:
|
||||
parts.append(f"{count} {key}")
|
||||
# Counted once, for the heading and the total alike: a file bundled
|
||||
# with its emulator is not one to source.
|
||||
emulator_bundled = sum(
|
||||
1 for g in data["gap_details"]
|
||||
if g.get("bundled") and g.get("source") == "missing"
|
||||
)
|
||||
counts = {key: data.get(f"gap_{key}", 0) for key in totals}
|
||||
counts["missing"] -= emulator_bundled
|
||||
counts["bundled"] = emulator_bundled
|
||||
parts = [f"{count} {key}" for key, count in counts.items() if count]
|
||||
status = ", ".join(parts) if parts else "OK"
|
||||
|
||||
systems = ", ".join(data["systems"])
|
||||
@@ -840,13 +910,8 @@ def print_report(
|
||||
|
||||
total_gaps += gaps
|
||||
for key in totals:
|
||||
totals[key] += data.get(f"gap_{key}", 0)
|
||||
emulator_bundled = sum(
|
||||
1 for g in data["gap_details"]
|
||||
if g.get("bundled") and g.get("source") == "missing"
|
||||
)
|
||||
totals[key] += counts[key]
|
||||
bundled += emulator_bundled
|
||||
totals["missing"] -= emulator_bundled
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Total: {total_gaps} undeclared files across {scope}")
|
||||
@@ -957,6 +1022,7 @@ def main() -> None:
|
||||
profiles, declared, db, plat_data_dirs, data_names,
|
||||
all_declared=declared_names(args.platforms_dir, selected, db),
|
||||
standalone_cores=standalone_cores,
|
||||
folded_slots=folded_declared_slots(args.platforms_dir, selected, db),
|
||||
)
|
||||
|
||||
if args.json:
|
||||
|
||||
@@ -94,6 +94,7 @@ from validation import (
|
||||
_build_validation_index,
|
||||
deliverable,
|
||||
filter_files_by_mode,
|
||||
with_archive_members,
|
||||
)
|
||||
|
||||
yaml = require_yaml()
|
||||
@@ -121,6 +122,20 @@ def _member_compression(arcname: str) -> int:
|
||||
return zipfile.ZIP_DEFLATED
|
||||
|
||||
|
||||
class PackRecord:
|
||||
"""A pack build kept in memory: the paths it writes, no archive.
|
||||
|
||||
``members`` is every path written, ``files`` the ones the build counts as
|
||||
the profile's files (archives and loose files, not trees or notes). The
|
||||
pipeline builds every emulator pack this way and compares ``files`` with
|
||||
what verify --emulator credits, without reading or compressing a byte.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.members: list[str] = []
|
||||
self.files: list[str] = []
|
||||
|
||||
|
||||
def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> None:
|
||||
"""Add a file to the pack with metadata that does not depend on the build.
|
||||
|
||||
@@ -132,6 +147,9 @@ def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> Non
|
||||
two consecutive builds. The content is streamed rather than read whole,
|
||||
because some members are firmware images of several hundred megabytes.
|
||||
"""
|
||||
if isinstance(zf, PackRecord):
|
||||
zf.members.append(arcname)
|
||||
return
|
||||
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
|
||||
info.compress_type = _member_compression(arcname)
|
||||
# The executable bit is the one permission worth carrying: the RetroDECK
|
||||
@@ -158,6 +176,9 @@ def _write_generated_member(zf: zipfile.ZipFile, arcname: str, text: str) -> Non
|
||||
a different hash on every run. Pinning them to the epoch the archive
|
||||
rebuilder already uses makes a pack a function of its inputs alone.
|
||||
"""
|
||||
if isinstance(zf, PackRecord):
|
||||
zf.members.append(arcname)
|
||||
return
|
||||
info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME)
|
||||
info.compress_type = zipfile.ZIP_DEFLATED
|
||||
info.external_attr = 0o100644 << 16
|
||||
@@ -934,6 +955,13 @@ def generate_pack(
|
||||
) as tmp:
|
||||
tmp_path = tmp.name
|
||||
|
||||
# The slot is the platform's file whether the download
|
||||
# arrives or not: verify cannot fetch, and a core file
|
||||
# filling it on a failed download made the two differ.
|
||||
seen_destinations.add(dedup_key)
|
||||
_register_path(dedup_key, seen_destinations, seen_parents)
|
||||
if case_insensitive:
|
||||
seen_lower.add(dedup_key.lower())
|
||||
try:
|
||||
if download_external(file_entry, tmp_path):
|
||||
extract = file_entry.get("extract", False)
|
||||
@@ -941,10 +969,6 @@ def generate_pack(
|
||||
_extract_zip_to_archive(tmp_path, _flat(full_dest, base_dest, flatten), zf)
|
||||
else:
|
||||
_add_pack_member(zf, tmp_path, _flat(full_dest, base_dest, flatten))
|
||||
seen_destinations.add(dedup_key)
|
||||
_register_path(dedup_key, seen_destinations, seen_parents)
|
||||
if case_insensitive:
|
||||
seen_lower.add(dedup_key.lower())
|
||||
file_status.setdefault(dedup_key, "ok")
|
||||
total_files += 1
|
||||
else:
|
||||
@@ -1273,6 +1297,9 @@ def _add_zip_to_pack(
|
||||
declare the hash of a ROM inside the ZIP (checkInsideZip), which a
|
||||
content-preserving rebuild does not affect.
|
||||
"""
|
||||
if isinstance(target_zf, PackRecord):
|
||||
target_zf.members.append(dest_path)
|
||||
return
|
||||
if (
|
||||
file_entry
|
||||
and not file_entry.get("zipped_file")
|
||||
@@ -1414,6 +1441,7 @@ def generate_emulator_pack(
|
||||
pack_name: str | None = None,
|
||||
system_ids: list[str] | None = None,
|
||||
final_dir: str | None = None,
|
||||
record: PackRecord | None = None,
|
||||
) -> str | None:
|
||||
"""Generate a ZIP pack for specific emulator profiles.
|
||||
|
||||
@@ -1422,6 +1450,8 @@ def generate_emulator_pack(
|
||||
the 255 bytes a file name may hold. `system_ids` keeps only the entries
|
||||
of those systems, as a platform pack narrows core extras. `final_dir`
|
||||
is where the caller moves the archive, the path the report names.
|
||||
`record` builds the pack in memory instead (PackRecord): nothing is
|
||||
written or read, and the paths are kept on it.
|
||||
"""
|
||||
all_profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
|
||||
if zip_contents is None:
|
||||
@@ -1446,7 +1476,8 @@ def generate_emulator_pack(
|
||||
display_names, [n for n, _p in selected], f"{narrow_tags}_BIOS_Pack.zip"
|
||||
)
|
||||
zip_path = os.path.join(output_dir, zip_name)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
if record is None:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
region_drops: set[str] = (
|
||||
emulator_region_drops(selected, standalone, regions, required_only, db)
|
||||
@@ -1484,7 +1515,10 @@ def generate_emulator_pack(
|
||||
platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms")
|
||||
)
|
||||
|
||||
with _writing_pack(zip_path) as zf:
|
||||
with (
|
||||
contextlib.nullcontext(record) if record is not None
|
||||
else _writing_pack(zip_path)
|
||||
) as zf:
|
||||
for emu_name, profile in sorted(selected):
|
||||
pack_structure = profile.get("pack_structure")
|
||||
files = expand_agnostic_entries(
|
||||
@@ -1549,6 +1583,8 @@ def generate_emulator_pack(
|
||||
seen_lower.add(archive_dest.lower())
|
||||
packed_at[archive_dest.lower()] = local_path
|
||||
total_files += 1
|
||||
if record is not None:
|
||||
record.files.append(archive_dest)
|
||||
else:
|
||||
report_missing(archive_name, archive_dest)
|
||||
|
||||
@@ -1608,7 +1644,7 @@ def generate_emulator_pack(
|
||||
refused_path: str | None = None
|
||||
for fe in alternatives:
|
||||
local_path, status = resolve_file(
|
||||
{**fe, "source_profile": emu_name},
|
||||
{**with_archive_members(fe, files), "source_profile": emu_name},
|
||||
db,
|
||||
bios_dir,
|
||||
zip_contents,
|
||||
@@ -1676,12 +1712,16 @@ def generate_emulator_pack(
|
||||
) as tmp:
|
||||
tmp_path = tmp.name
|
||||
try:
|
||||
if download_external(fe, tmp_path):
|
||||
# Built in memory, nothing is fetched: verify --emulator
|
||||
# cannot fetch either and reports the file missing.
|
||||
if record is None and download_external(fe, tmp_path):
|
||||
_add_pack_member(zf, tmp_path, dest)
|
||||
seen_destinations.add(dest)
|
||||
_register_path(dest, seen_destinations, seen_parents)
|
||||
seen_lower.add(dest.lower())
|
||||
total_files += 1
|
||||
if record is not None:
|
||||
record.files.append(dest)
|
||||
else:
|
||||
missing_files.append(fe["name"])
|
||||
finally:
|
||||
@@ -1710,6 +1750,8 @@ def generate_emulator_pack(
|
||||
seen_lower.add(dest.lower())
|
||||
packed_at[dest.lower()] = local_path
|
||||
total_files += 1
|
||||
if record is not None:
|
||||
record.files.append(dest)
|
||||
|
||||
# The trees after the files, as in a platform pack: a dump the
|
||||
# profile declares wins over the copy a tree bundles at the same
|
||||
@@ -1742,7 +1784,9 @@ def generate_emulator_pack(
|
||||
)
|
||||
|
||||
# Remove empty ZIP (no files packed and no missing = nothing to ship)
|
||||
if total_files + data_files == 0 and not (missing_files or rejected_files or instruction_files or unsourceable_files):
|
||||
if record is None and total_files + data_files == 0 and not (
|
||||
missing_files or rejected_files or instruction_files or unsourceable_files
|
||||
):
|
||||
os.unlink(zip_path)
|
||||
|
||||
# Report. The members of data trees are counted apart: verify --emulator
|
||||
|
||||
+128
-16
@@ -9,7 +9,7 @@ from __future__ import annotations
|
||||
from collections.abc import Iterable
|
||||
|
||||
from common import PROFILE_IDENTITY_FIELDS, _norm_system_id
|
||||
from common import archive_members
|
||||
from common import archive_members, holds_required_members
|
||||
from common import declared_root, expand_platform_declared_names, platform_declarations
|
||||
from common import load_emulator_profiles
|
||||
from common import resolution_is_hash_exact
|
||||
@@ -209,7 +209,7 @@ def hidden_extras(
|
||||
emulators_dir: str,
|
||||
db: dict,
|
||||
profiles: dict,
|
||||
shipped: dict[str, str],
|
||||
shipped: dict[str, str | None],
|
||||
target_cores: set[str] | None = None,
|
||||
region_drops: set[str] | None = None,
|
||||
placed: set[str] | None = None,
|
||||
@@ -229,7 +229,8 @@ def hidden_extras(
|
||||
while the report counted it in the pack. A twin carrying the same bytes
|
||||
serves the core on the filesystem that folds the names, so it is not a
|
||||
file left out. ``shipped`` maps the full destination of each platform
|
||||
file the pack carries to its local copy. ``placed``, when given,
|
||||
file the pack carries to its local copy, or to None for a slot it keeps
|
||||
without one (user_provided, external). ``placed``, when given,
|
||||
receives the full destination of each held extra the pack carries.
|
||||
``resolver`` is the builder's lookup. An extra it cannot resolve is
|
||||
skipped and takes no path, as the builder skips it.
|
||||
@@ -879,11 +880,17 @@ def _collect_emulator_extras(
|
||||
undeclared = undeclared_report(
|
||||
config, emulators_dir, db, emu_profiles, target_cores, include_all
|
||||
)
|
||||
# A destination two cores claim goes to the one that names its content:
|
||||
# A destination two cores claim goes to the strongest proof of what the
|
||||
# collection holds for it, then to the one that names its content:
|
||||
# hypseus-singe asks for gamecontrollerdb.txt by name alone and took the
|
||||
# BIOS root before Solarus, which pins its own by sha1. Profile order
|
||||
# still decides between claims of the same kind.
|
||||
# BIOS root before Solarus, which pins its own by sha1, and fbalpha2012's
|
||||
# hashless hiscore.dat took RetroDECK's over mame2015's sha1. Profile
|
||||
# order decides only between claims of equal proof.
|
||||
undeclared = sorted(undeclared, key=lambda u: not _names_content(u))
|
||||
rank = _ClaimRanks(
|
||||
undeclared, placement, db, bool(config.get("case_insensitive_fs", False))
|
||||
)
|
||||
undeclared = rank.ordered(undeclared)
|
||||
extras = []
|
||||
seen_dests: set[str] = set(seen)
|
||||
# Destination -> the extra that took it, to record later claimants on.
|
||||
@@ -922,12 +929,18 @@ def _collect_emulator_extras(
|
||||
claimants[(u.get("emulator", ""), u.get("name", ""), u.get("path") or "")] = (
|
||||
sanitize_pack_path(dest)
|
||||
)
|
||||
extra = _report_extra(u, name, dest)
|
||||
claims = [_report_extra(c, name, dest) for c in u.get("other_claims", [])]
|
||||
# Where the builder meets each claim: the report lists the profiles in
|
||||
# order, archives after the loose files.
|
||||
for item in (extra, *claims):
|
||||
item["claim_order"] = (1, 1 if u.get("archive") else 0, item["source_profile"])
|
||||
candidates = [
|
||||
_report_extra(c, name, dest) for c in (u, *u.get("other_claims", []))
|
||||
]
|
||||
# Where the builder meets each claim: the strongest proof first, the
|
||||
# report's profile order between equals, archives after loose files.
|
||||
for item in candidates:
|
||||
item["claim_order"] = (
|
||||
1, 1 if u.get("archive") else 0, -rank.of_extra(item, full_dest),
|
||||
item["source_profile"],
|
||||
)
|
||||
candidates.sort(key=lambda item: item["claim_order"])
|
||||
extra, claims = candidates[0], candidates[1:]
|
||||
if not _take(extra, full_dest, seen_dests, taken):
|
||||
for claim in claims:
|
||||
_another_claim(taken.get(full_dest), claim)
|
||||
@@ -1052,6 +1065,82 @@ def _collect_emulator_extras(
|
||||
|
||||
return extras
|
||||
|
||||
class _ClaimRanks:
|
||||
"""How strongly each claim on a contested destination proves its file.
|
||||
|
||||
2 a declared hash the collection holds, 1 a path or every member the
|
||||
core requires inside an archive, 0 a name, -1 a copy
|
||||
contradicting the hash the claim declares, -2 nothing held. Resolved only
|
||||
where two claims meet: a destination one core asks for has no contest.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, undeclared: list[dict], placement: ExtraPlacement, db: dict, fold: bool,
|
||||
) -> None:
|
||||
self._db = db
|
||||
# Two spellings of one path are one file where the filesystem folds case.
|
||||
self._key = (lambda dest: dest.lower()) if fold else (lambda dest: dest)
|
||||
self._placed: dict[int, tuple[str, str] | None] = {}
|
||||
counts: dict[str, int] = {}
|
||||
for u in undeclared:
|
||||
placed = (
|
||||
placement.place(u, _extra_dest(u))
|
||||
if u.get("in_repo") and not u.get("bundled") else None
|
||||
)
|
||||
self._placed[id(u)] = placed
|
||||
if placed is not None:
|
||||
key = self._key(placed[1])
|
||||
counts[key] = counts.get(key, 0) + 1 + len(u.get("other_claims", []))
|
||||
self._contested = {dest for dest, n in counts.items() if n > 1}
|
||||
self._cache: dict[tuple, int] = {}
|
||||
|
||||
def of_extra(self, extra: dict, full_dest: str) -> int:
|
||||
if self._key(full_dest) not in self._contested:
|
||||
return 0
|
||||
key = (extra.get("source_profile"), extra.get("source_name"),
|
||||
extra.get("source_path"), full_dest)
|
||||
if key not in self._cache:
|
||||
path, status = resolve_local_file(
|
||||
extra, self._db, dest_hint=extra.get("destination", "")
|
||||
)
|
||||
self._cache[key] = (
|
||||
-2 if not path
|
||||
else -1 if status == "hash_mismatch"
|
||||
else 2 if resolution_is_hash_exact(status)
|
||||
else 1 if status == "path_exact" or (
|
||||
extra.get("members") and holds_required_members(path, extra["members"])
|
||||
)
|
||||
else 0
|
||||
)
|
||||
return self._cache[key]
|
||||
|
||||
def ordered(self, undeclared: list[dict]) -> list[dict]:
|
||||
"""The entries in their order, the claims on one contested path
|
||||
reordered among themselves by proof. Nothing else moves, so the
|
||||
pack and the manifest keep their order."""
|
||||
first: dict[str, int] = {}
|
||||
keys: list[tuple[int, int, int]] = []
|
||||
for position, u in enumerate(undeclared):
|
||||
placed = self._placed.get(id(u))
|
||||
if placed is None or self._key(placed[1]) not in self._contested:
|
||||
keys.append((position, 0, position))
|
||||
continue
|
||||
group = first.setdefault(self._key(placed[1]), position)
|
||||
keys.append((group, -self.of_entry(u), position))
|
||||
order = sorted(range(len(undeclared)), key=keys.__getitem__)
|
||||
return [undeclared[i] for i in order]
|
||||
|
||||
def of_entry(self, u: dict) -> int:
|
||||
placed = self._placed.get(id(u))
|
||||
if placed is None or self._key(placed[1]) not in self._contested:
|
||||
return 0
|
||||
name = u.get("archive") or u["name"]
|
||||
return max(
|
||||
self.of_extra(_report_extra(c, name, placed[0]), placed[1])
|
||||
for c in (u, *u.get("other_claims", []))
|
||||
)
|
||||
|
||||
|
||||
def _names_content(entry: dict) -> bool:
|
||||
"""Whether a report entry declares the bytes it wants, not only a name."""
|
||||
return any(entry.get(field) for field in ("sha1", "md5", "sha256", "crc32"))
|
||||
@@ -1316,11 +1405,12 @@ def emulator_region_drops(
|
||||
"""
|
||||
import region as region_mod
|
||||
from packpaths import _resolve_destination
|
||||
from validation import filter_files_by_mode
|
||||
from validation import _build_validation_index, filter_files_by_mode
|
||||
|
||||
region_index = region_mod.build_region_index(dict(selected))
|
||||
region_groups: dict[str, list[tuple[str, str]]] = {}
|
||||
held: set[str] | None = set() if db is not None else None
|
||||
validation_index = _build_validation_index(dict(selected)) if db is not None else {}
|
||||
for emu_name, profile in sorted(selected):
|
||||
# Decided over what the pack keeps, as on the platform side: an
|
||||
# optional winner removed afterwards left its slot empty.
|
||||
@@ -1332,13 +1422,35 @@ def emulator_region_drops(
|
||||
region_groups.setdefault(group_id, []).append(
|
||||
(dest, fe.get("name", ""))
|
||||
)
|
||||
if held is not None and resolve_local_file(
|
||||
dict(fe, source_profile=emu_name), db, dest_hint=dest
|
||||
)[0]:
|
||||
if held is not None and _emulator_ships(
|
||||
fe, emu_name, profile, db, dest, validation_index
|
||||
):
|
||||
held.add(dest)
|
||||
return region_mod.resolve_region_drops(region_groups, region_index, regions, held)
|
||||
|
||||
|
||||
def _emulator_ships(
|
||||
file_entry: dict, emu_name: str, profile: dict, db: dict, dest: str,
|
||||
validation_index: dict,
|
||||
) -> bool:
|
||||
"""Whether the emulator pack ships this entry: held, and not refused by
|
||||
the core's own check, the test _ships applies on the platform side.
|
||||
"""
|
||||
from validation import validated_choice, with_archive_members
|
||||
|
||||
local, _status = resolve_local_file(
|
||||
{**with_archive_members(file_entry, profile.get("files", [])),
|
||||
"source_profile": emu_name},
|
||||
db, dest_hint=dest,
|
||||
)
|
||||
if not local:
|
||||
return False
|
||||
_chosen, disagreement = validated_choice(
|
||||
file_entry, local, db, validation_index, "bios", None, emulators={emu_name},
|
||||
)
|
||||
return disagreement is None
|
||||
|
||||
|
||||
def _emulator_region_group(
|
||||
emu_name: str, profile: dict, file_entry: dict, dest: str
|
||||
) -> str:
|
||||
|
||||
+174
-3
@@ -21,7 +21,7 @@ Steps, numbered as the run prints them:
|
||||
4b generate_pack.py --manifest install manifests
|
||||
4c --manifest-targets per-target core lists
|
||||
4d bundled_only.py --strict copies only an emulator install provides
|
||||
5 consistency check verify counts == pack counts
|
||||
5 consistency check verify == pack: counts, paths, every emulator pack
|
||||
6 --verify-packs pack integrity, extraction and hashes
|
||||
7 generate_readme.py README.md and CONTRIBUTING.md
|
||||
8 generate_site.py MkDocs pages
|
||||
@@ -41,13 +41,16 @@ Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import atexit
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
@@ -382,6 +385,159 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
|
||||
return all_ok
|
||||
|
||||
|
||||
def parse_pack_paths(output: str) -> dict[str, str]:
|
||||
"""The archive each full pack was written to, per pack label."""
|
||||
paths: dict[str, str] = {}
|
||||
current_label = ""
|
||||
for line in output.splitlines():
|
||||
m = re.match(r"Generating (?:shared )?pack for (.+)\.\.\.", line)
|
||||
if m:
|
||||
current_label = _full_pack_label(m.group(1).strip())
|
||||
continue
|
||||
written = re.match(r"^\s+(\S.*?\.zip): \d+ files packed", line)
|
||||
if current_label and written:
|
||||
paths[current_label] = written.group(1)
|
||||
return paths
|
||||
|
||||
|
||||
# Members the builder writes for the reader rather than for a core.
|
||||
_PACK_NOTES = re.compile(r"(^README\.txt$|^manifest\.json$|(^|/)(INSTRUCTIONS|RENAMED)_[^/]*\.txt$)")
|
||||
|
||||
|
||||
def _tree_members(platform: str, platforms_dir: str) -> dict[str, int]:
|
||||
"""Archive path -> size of every file the platform's data trees can add."""
|
||||
import generate_pack # noqa: PLC0415
|
||||
from common import load_platform_config # noqa: PLC0415
|
||||
from packpaths import _flat # noqa: PLC0415
|
||||
|
||||
config = load_platform_config(platform, platforms_dir)
|
||||
base_dest = config.get("base_destination", "")
|
||||
members: dict[str, int] = {}
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
for src, full in generate_pack._data_directory_members(
|
||||
config.get("systems", {}), load_data_dir_registry(platforms_dir),
|
||||
platform, base_dest, False, set(), set(), set(),
|
||||
):
|
||||
members[_flat(full, base_dest, True)] = os.path.getsize(src)
|
||||
return members
|
||||
|
||||
|
||||
def _set_difference_lines(label: str, credited: set[str], written: set[str]) -> list[str]:
|
||||
"""The paths one side has and the other lacks, ten of each at most."""
|
||||
lines = []
|
||||
for side, paths in (("verify only", credited - written), ("pack only", written - credited)):
|
||||
if paths:
|
||||
shown = ", ".join(sorted(paths)[:10])
|
||||
more = f" (+{len(paths) - 10})" if len(paths) > 10 else ""
|
||||
lines.append(f" {label}: {side} {len(paths)}: {shown}{more}")
|
||||
return lines
|
||||
|
||||
|
||||
def check_pack_destinations(
|
||||
destinations_file: str, pack_output: str, platforms_dir: str = "platforms",
|
||||
) -> bool:
|
||||
"""Compare the paths verify credits with the members of each archive.
|
||||
|
||||
Equal counts hid a file credited on one side and another written on the
|
||||
other. The trees and the notes are the archive's own: a tree member is
|
||||
set aside when its path and size are the cached file's and verify does
|
||||
not credit the path.
|
||||
"""
|
||||
import json # noqa: PLC0415
|
||||
import zipfile # noqa: PLC0415
|
||||
|
||||
print("\n--- 5/8 consistency check: destinations ---")
|
||||
try:
|
||||
credited_by = json.loads(Path(destinations_file).read_text())
|
||||
except (OSError, ValueError) as exc:
|
||||
print(f" no destinations from verify ({exc})")
|
||||
return False
|
||||
paths = parse_pack_paths(pack_output)
|
||||
if not paths:
|
||||
print(" no archive named in the pack output")
|
||||
return False
|
||||
all_ok = True
|
||||
for label, zip_path in sorted(paths.items()):
|
||||
names = [n for n in credited_by if _match_key(n) & _match_key(label)]
|
||||
if not names:
|
||||
print(f" {label}: no verify destinations")
|
||||
all_ok = False
|
||||
continue
|
||||
entry = credited_by[names[0]]
|
||||
credited = set(entry["pack_destinations"])
|
||||
trees = _tree_members(entry["platform"], platforms_dir)
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
written = {
|
||||
info.filename for info in zf.infolist()
|
||||
if not info.is_dir()
|
||||
and not _PACK_NOTES.search(info.filename)
|
||||
and not (
|
||||
trees.get(info.filename) == info.file_size
|
||||
and info.filename not in credited
|
||||
)
|
||||
}
|
||||
lines = _set_difference_lines(label, credited, written)
|
||||
if lines:
|
||||
print("\n".join(lines))
|
||||
all_ok = False
|
||||
else:
|
||||
print(f" {label}: {len(credited)} paths, the same on both sides")
|
||||
return all_ok
|
||||
|
||||
|
||||
def check_emulator_packs(
|
||||
emulators_dir: str = "emulators", platforms_dir: str = "platforms",
|
||||
db_path: str = "database.json",
|
||||
) -> bool:
|
||||
"""Build every emulator pack in memory and compare it with verify --emulator.
|
||||
|
||||
Each profile in each build it has (libretro, standalone): the paths the
|
||||
pack fills with the profile's files against the ones verify credits. No
|
||||
archive is written and no file read, so all of them take seconds.
|
||||
"""
|
||||
import generate_pack # noqa: PLC0415
|
||||
import verify # noqa: PLC0415
|
||||
from common import ( # noqa: PLC0415
|
||||
ProfileSelectionError,
|
||||
build_zip_contents_index,
|
||||
load_database,
|
||||
load_emulator_profiles,
|
||||
select_emulator_profiles,
|
||||
)
|
||||
|
||||
print("\n--- 5/8 consistency check: emulator packs ---")
|
||||
db = load_database(db_path)
|
||||
zip_contents = build_zip_contents_index(db)
|
||||
profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
|
||||
all_ok = True
|
||||
compared = 0
|
||||
for name in sorted(profiles):
|
||||
for standalone in (False, True):
|
||||
try:
|
||||
select_emulator_profiles([name], profiles, standalone)
|
||||
except ProfileSelectionError:
|
||||
continue
|
||||
record = generate_pack.PackRecord()
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
generate_pack.generate_emulator_pack(
|
||||
[name], emulators_dir, db, "bios", "", standalone, zip_contents,
|
||||
offline=True, platforms_dir=platforms_dir, record=record,
|
||||
)
|
||||
report = verify.verify_emulator(
|
||||
[name], emulators_dir, db, standalone, platforms_dir=platforms_dir,
|
||||
)
|
||||
compared += 1
|
||||
label = f"{name}{' --standalone' if standalone else ''}"
|
||||
lines = _set_difference_lines(
|
||||
label, set(report["pack_destinations"]), set(record.files)
|
||||
)
|
||||
if lines:
|
||||
print("\n".join(lines))
|
||||
all_ok = False
|
||||
print(f" {compared} emulator packs compared, {'all equal' if all_ok else 'differences above'}")
|
||||
return all_ok
|
||||
|
||||
|
||||
class _Skipped:
|
||||
"""A step that did not run. Truthy, so it never fails the pipeline, and
|
||||
distinct, so the summary does not report it as work that was done."""
|
||||
@@ -681,8 +837,19 @@ def main() -> None:
|
||||
# exclusive.
|
||||
hold_data_dirs(held, load_data_dir_registry("platforms"))
|
||||
|
||||
# Step 3: Verify
|
||||
verify_cmd = [sys.executable, "scripts/verify.py", "--all", "--strict"]
|
||||
# Step 3: Verify. The paths each pack holds by this report go to a file
|
||||
# step 5 compares with the archives.
|
||||
scratch = Path(__file__).resolve().parent.parent / "tmp"
|
||||
scratch.mkdir(exist_ok=True)
|
||||
fd, destinations_file = tempfile.mkstemp(
|
||||
prefix="verify-destinations.", suffix=".json", dir=scratch
|
||||
)
|
||||
os.close(fd)
|
||||
atexit.register(Path(destinations_file).unlink, missing_ok=True)
|
||||
verify_cmd = [
|
||||
sys.executable, "scripts/verify.py", "--all", "--strict",
|
||||
"--destinations", destinations_file,
|
||||
]
|
||||
if args.include_archived:
|
||||
verify_cmd.append("--include-archived")
|
||||
if args.target:
|
||||
@@ -833,6 +1000,10 @@ def main() -> None:
|
||||
results["consistency"] = SKIPPED
|
||||
elif pack_output and verify_output:
|
||||
ok = check_consistency(verify_output, pack_output)
|
||||
# Counts can agree while the paths differ, both ways at once.
|
||||
ok = check_pack_destinations(destinations_file, pack_output) and ok
|
||||
ok = check_emulator_packs() and ok
|
||||
print(f"--- 5/8 consistency check, counts and paths: {'OK' if ok else 'FAILED'} ---")
|
||||
results["consistency"] = ok
|
||||
all_ok = all_ok and ok
|
||||
else:
|
||||
|
||||
+56
-22
@@ -1,8 +1,8 @@
|
||||
"""Who decides what goes at a destination.
|
||||
|
||||
A destination is a slot. Two layers can claim it: the platform YAML, scraped
|
||||
from what the frontend declares and therefore able to carry an upstream error,
|
||||
and the emulator profile, read from the emulator's own source. Until this
|
||||
from what the frontend declares, which can pin another file than the emulator
|
||||
reads, and the emulator profile, read from the emulator's own source. Until this
|
||||
module existed nothing compared the two: the builder resolves the platform
|
||||
entry first and drops the profile entry on a bare filename match, so a profile
|
||||
that names the right file for a slot could never win and never even be heard.
|
||||
@@ -37,7 +37,7 @@ from common import (
|
||||
variant_base_name,
|
||||
)
|
||||
from packresolve import preferred_entries
|
||||
from validation import outside_gap_scope
|
||||
from validation import outside_gap_scope, with_archive_members
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from packextras import ExtraPlacement
|
||||
@@ -241,7 +241,10 @@ def profile_claims(
|
||||
full = placed[1]
|
||||
# The owner's own copy is the one the builder ships.
|
||||
local, status = resolve_local_file(
|
||||
{**entry, "source_profile": emu_name},
|
||||
{
|
||||
**with_archive_members(entry, profile.get("files") or []),
|
||||
"source_profile": emu_name,
|
||||
},
|
||||
db,
|
||||
zip_contents,
|
||||
dest_hint=dest,
|
||||
@@ -326,6 +329,29 @@ def _archive_claims(
|
||||
return claims
|
||||
|
||||
|
||||
def checked_by_path(mode: str, entry: dict, rule: str = "") -> bool:
|
||||
"""Whether the frontend checks this declaration by its path alone.
|
||||
|
||||
Whether a frontend reads the bytes is a property of the platform, but
|
||||
what it compares them with is the entry's: Batocera skips an empty md5
|
||||
(batocera-systems:1088), and EmuDeck declares neocdz.zip with no hash at
|
||||
all. Such an entry is a path, in an md5 platform as in an existence one.
|
||||
A member pinned inside an archive is content.
|
||||
"""
|
||||
if not nativemode.reads_file_contents(mode):
|
||||
return True
|
||||
if entry.get("zipped_file"):
|
||||
return False
|
||||
if rule == nativemode.SIZE_AND_ANY_HASH:
|
||||
fields: tuple[str, ...] = ("md5", "sha1", "crc32")
|
||||
else:
|
||||
fields = (nativemode.digest_algorithm(mode) or "",)
|
||||
return not any(
|
||||
parse_md5_list(entry.get(name)) if name == "md5" else entry.get(name)
|
||||
for name in fields
|
||||
)
|
||||
|
||||
|
||||
def _claim_rank(claim: Claim) -> int:
|
||||
"""How strongly a platform claim speaks for what the pack ships."""
|
||||
if resolution_is_hash_exact(claim.status):
|
||||
@@ -379,10 +405,10 @@ def _slot_claims(
|
||||
# matches, asserts nothing about content. Where the frontend never reads
|
||||
# the bytes, a proven profile claim then decides: RetroArch's neocdz.zip
|
||||
# pins an md5 nobody holds, and the primary shipped without the 000-lo.lo
|
||||
# geolith and FBNeo read inside it.
|
||||
path_only = not nativemode.reads_file_contents(
|
||||
config.get("verification_mode", "existence")
|
||||
)
|
||||
# geolith and FBNeo read inside it. Decided per entry: EmuDeck, an md5
|
||||
# platform, declares the same archive with no hash.
|
||||
mode = config.get("verification_mode", "existence")
|
||||
rule = config.get("hash_rule", "")
|
||||
by_slot: dict[str, list[Claim]] = {}
|
||||
for claim in profile_claims(
|
||||
profiles, db, base_dest, standalone_cores, zip_contents, data_dir_registry,
|
||||
@@ -398,7 +424,9 @@ def _slot_claims(
|
||||
):
|
||||
# A size proves a claim only against a file of another size.
|
||||
continue
|
||||
if not platform.is_proven and not (path_only and platform.local_path):
|
||||
if not platform.is_proven and not (
|
||||
platform.local_path and checked_by_path(mode, platform.entry, rule)
|
||||
):
|
||||
continue
|
||||
by_slot.setdefault(key, []).append(claim)
|
||||
|
||||
@@ -515,8 +543,8 @@ def slot_choices(
|
||||
recorded whole in the install manifest, so a target narrowed from the
|
||||
full manifest gets the bytes the target's own pack carries.
|
||||
"""
|
||||
if nativemode.reads_file_contents(config.get("verification_mode", "existence")):
|
||||
return {}
|
||||
mode = config.get("verification_mode", "existence")
|
||||
rule = config.get("hash_rule", "")
|
||||
platform_profiles = {
|
||||
name: profiles[name] for name in resolve_platform_cores(config, profiles)
|
||||
}
|
||||
@@ -532,6 +560,9 @@ def slot_choices(
|
||||
choices: dict[str, list[tuple[str, str | None]]] = {}
|
||||
for key, claims in sorted(by_slot.items()):
|
||||
platform = by_dest[key]
|
||||
# Where the frontend compares a hash, the platform's file stays.
|
||||
if not checked_by_path(mode, platform.entry, rule):
|
||||
continue
|
||||
agree = [_satisfies(c, platform, db) for c in claims]
|
||||
agreeing = [c for c, ok in zip(claims, agree) if ok]
|
||||
others = sorted(
|
||||
@@ -570,7 +601,9 @@ def _strongest(claims: list[Claim]) -> Claim:
|
||||
return max(claims, key=lambda claim: (_claim_rank(claim), -claims.index(claim)))
|
||||
|
||||
|
||||
def arbitrate(conflict: Conflict, mode: str, addressee: str = "platform") -> Decision:
|
||||
def arbitrate(
|
||||
conflict: Conflict, mode: str, addressee: str = "platform", rule: str = "",
|
||||
) -> Decision:
|
||||
"""Decide a contested destination for the pack being built.
|
||||
|
||||
A pack answers to whoever asked for it. A platform pack must leave the
|
||||
@@ -581,13 +614,13 @@ def arbitrate(conflict: Conflict, mode: str, addressee: str = "platform") -> Dec
|
||||
In ``existence`` mode the frontend only looks for a path, so the emulator's
|
||||
file satisfies both sides at once and there is nothing to trade away. In a
|
||||
content-checking mode the two answers cannot share one path, and the pack's
|
||||
addressee decides. The loss is reported rather than absorbed, because the
|
||||
cause is an upstream declaration that needs fixing at its source.
|
||||
addressee decides. The loss is reported rather than absorbed: only the
|
||||
platform's own list can change what its frontend accepts there.
|
||||
"""
|
||||
profile = _strongest(conflict.profile_claims)
|
||||
if addressee == "emulator":
|
||||
return Decision(conflict, profile, ADDRESSEE)
|
||||
if not nativemode.reads_file_contents(mode):
|
||||
if checked_by_path(mode, conflict.platform_claim.entry, rule):
|
||||
return Decision(conflict, profile, SERVES_BOTH)
|
||||
return Decision(conflict, conflict.platform_claim, FRONTEND_CHECKS_CONTENT)
|
||||
|
||||
@@ -626,10 +659,11 @@ def format_decision(decision: Decision) -> str:
|
||||
f"{conflict.destination}: serve {decision.winner.local_path}, "
|
||||
"the pack answers to the emulator"
|
||||
)
|
||||
expected = ", ".join(conflict.emulators) or "the profile"
|
||||
return (
|
||||
f"{conflict.destination}: keep {decision.winner.local_path}, the frontend "
|
||||
f"verifies content and would reject "
|
||||
f"{conflict.profile_claims[0].local_path}; upstream declaration is wrong"
|
||||
f"{conflict.profile_claims[0].local_path}, the file {expected} reads"
|
||||
)
|
||||
|
||||
|
||||
@@ -659,8 +693,8 @@ def find_collisions(
|
||||
One path holds one file, so whichever declaration the builder reaches
|
||||
first decides in silence. Declaring an archive several times to name its
|
||||
inner ROMs is the documented zipped_file pattern and resolves to one
|
||||
archive. Two declarations landing on two files is a contradiction the
|
||||
upstream list carries. RetroDECK aims a PC-88 disk subsystem ROM and a
|
||||
archive. Two declarations landing on two files name two files for one
|
||||
path. RetroDECK aims a PC-88 disk subsystem ROM and a
|
||||
CoCo disk ROM at one bios/disk.rom.
|
||||
"""
|
||||
fold = bool(config.get("case_insensitive_fs", False))
|
||||
@@ -974,7 +1008,7 @@ def main() -> int:
|
||||
mode = config.get("verification_mode", "existence")
|
||||
print(f"{platform}: {len(conflicts)} contradicted [{mode}]")
|
||||
for conflict in conflicts:
|
||||
decision = arbitrate(conflict, mode)
|
||||
decision = arbitrate(conflict, mode, rule=config.get("hash_rule", ""))
|
||||
fixable += decision.serves_both
|
||||
print(f" {format_decision(decision)}")
|
||||
total = sum(len(c) for c in found.values())
|
||||
@@ -987,13 +1021,13 @@ def main() -> int:
|
||||
doubled = sum(len(c) for c in collided.values())
|
||||
print(
|
||||
f"\n{total} contested destinations. {fixable} the pack settles on "
|
||||
f"its own, {total - fixable} rest on an upstream declaration. "
|
||||
f"its own, {total - fixable} follow the platform's list. "
|
||||
f"{doubled} destinations declared twice."
|
||||
)
|
||||
if not args.strict and (total or doubled):
|
||||
print(
|
||||
"Reported, not failed: the remainder needs the upstream list "
|
||||
"corrected, which no build can do. Use --strict to gate on them."
|
||||
"Reported, not failed: the remainder changes when the platform's "
|
||||
"list does, which no build can do. Use --strict to gate on them."
|
||||
)
|
||||
|
||||
return 1 if ((found or collided) and args.strict) else 0
|
||||
|
||||
+34
-1
@@ -13,6 +13,7 @@ from collections.abc import Callable, Iterator, Mapping
|
||||
|
||||
from common import (
|
||||
adler32_of,
|
||||
archive_members,
|
||||
compute_hashes,
|
||||
name_match_size_ok,
|
||||
sanitize_pack_path,
|
||||
@@ -464,11 +465,25 @@ def outside_gap_scope(file_entry: dict, is_standalone: bool) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def case_folded_slots(
|
||||
config: dict, declarations: Mapping[str, list[tuple[int | None, str]]],
|
||||
) -> frozenset[str]:
|
||||
"""The slots a platform whose filesystem folds case fills, any spelling.
|
||||
|
||||
On RetroBat the 32X_G_BIOS.BIN it declares is the file BlastEm opens as
|
||||
32X_G_BIOS.bin. On Lakka bios.col and BIOS.col are two files: nothing.
|
||||
"""
|
||||
if not config.get("case_insensitive_fs", False):
|
||||
return frozenset()
|
||||
return frozenset(where for places in declarations.values() for _size, where in places)
|
||||
|
||||
|
||||
def declaration_settles(
|
||||
file_entry: dict,
|
||||
fname: str,
|
||||
declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str],
|
||||
dest: str = "",
|
||||
folded_slots: frozenset[str] = frozenset(),
|
||||
) -> bool:
|
||||
"""Whether a platform declaration meets a core's requirement.
|
||||
|
||||
@@ -485,12 +500,16 @@ def declaration_settles(
|
||||
carries no slot or size and answers by name.
|
||||
|
||||
Read by verify and by the gap report, which settled on the name alone
|
||||
and left DOSBox's SC-55 ROMs off the gaps page.
|
||||
and left DOSBox's SC-55 ROMs off the gaps page. ``folded_slots``
|
||||
(case_folded_slots) are slots a declaration fills in another spelling on
|
||||
a filesystem that folds case: the core opens that file there.
|
||||
"""
|
||||
if file_entry.get("archive") in declared_names:
|
||||
return True
|
||||
slot = sanitize_pack_path(dest or fname).lower()
|
||||
directory = slot if dest.endswith("/") else slot.rpartition("/")[0]
|
||||
if not dest.endswith("/") and slot in folded_slots:
|
||||
return True
|
||||
for name in (fname, *file_entry.get("aliases", [])):
|
||||
if name not in declared_names:
|
||||
continue
|
||||
@@ -512,6 +531,20 @@ def declaration_settles(
|
||||
return False
|
||||
|
||||
|
||||
def with_archive_members(entry: dict, files: list[dict]) -> dict:
|
||||
"""A loose entry resolved as the archive its profile reads members from.
|
||||
|
||||
gngeopi declares neogeo.zip and, apart, the ROMs it opens inside it. The
|
||||
members decide which same-named copy is the archive: resolved by name
|
||||
alone, the loose entry took another copy than the archive pass and the
|
||||
profile blocked its own archive.
|
||||
"""
|
||||
if entry.get("archive"):
|
||||
return entry
|
||||
members = archive_members(entry.get("name", ""), files)
|
||||
return {**entry, "members": members} if members else entry
|
||||
|
||||
|
||||
def opens_by_name(file_entry: dict, profile: dict, standalone: bool = False) -> bool:
|
||||
"""Whether the code opens this entry by its name.
|
||||
|
||||
|
||||
+118
-10
@@ -30,6 +30,7 @@ import json
|
||||
import os
|
||||
import sys
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
|
||||
import slots
|
||||
|
||||
@@ -85,6 +86,7 @@ from nativemode import (
|
||||
)
|
||||
from validation import (
|
||||
baseline_verdict,
|
||||
case_folded_slots,
|
||||
declaration_settles,
|
||||
frontend_digest_matches,
|
||||
existence_discrepancy,
|
||||
@@ -100,6 +102,7 @@ from validation import (
|
||||
installed_with_emulator,
|
||||
unread_on,
|
||||
outside_gap_scope,
|
||||
with_archive_members,
|
||||
)
|
||||
|
||||
DEFAULT_DB = "database.json"
|
||||
@@ -442,6 +445,7 @@ def _candidate_verdict(
|
||||
include_all: bool,
|
||||
declared_names: Mapping[str, list[tuple[int | None, str]]] | set[str],
|
||||
dest: str = "",
|
||||
folded_slots: frozenset[str] = frozenset(),
|
||||
) -> str:
|
||||
"""Whether a profile entry can be a gap, and whether it is settled.
|
||||
|
||||
@@ -457,7 +461,7 @@ def _candidate_verdict(
|
||||
if outside_gap_scope(file_entry, is_standalone):
|
||||
return "skip"
|
||||
if not include_all and declaration_settles(
|
||||
file_entry, fname, declared_names, dest
|
||||
file_entry, fname, declared_names, dest, folded_slots
|
||||
):
|
||||
return "settled"
|
||||
return "keep"
|
||||
@@ -501,6 +505,7 @@ def find_undeclared_files(
|
||||
"""
|
||||
if declared_names is None:
|
||||
declared_names = platform_declarations(config, db)
|
||||
folded_slots = case_folded_slots(config, declared_names)
|
||||
|
||||
# Whether the builder drops a file whose local copy contradicts its
|
||||
# declared hash, which decides if such a copy counts as held here.
|
||||
@@ -590,7 +595,8 @@ def find_undeclared_files(
|
||||
if duplicate and (first is None or first["profile"] == emu_name):
|
||||
continue
|
||||
verdict = _candidate_verdict(
|
||||
f, fname, is_standalone, include_all, declared_names, dest
|
||||
f, fname, is_standalone, include_all, declared_names, dest,
|
||||
folded_slots,
|
||||
)
|
||||
if verdict == "settled":
|
||||
seen_files.add(seen_key)
|
||||
@@ -707,6 +713,9 @@ def find_undeclared_files(
|
||||
}
|
||||
if f.get("contents"):
|
||||
entry["contents"] = f["contents"]
|
||||
members = with_archive_members(f, profile.get("files", [])).get("members")
|
||||
if members:
|
||||
entry["members"] = members
|
||||
if launcher_layer:
|
||||
launcher_candidates.append((emu_name, is_standalone, f, entry))
|
||||
if duplicate:
|
||||
@@ -879,7 +888,7 @@ def _mark_hidden(
|
||||
emulators_dir: str,
|
||||
db: dict,
|
||||
emu_profiles: dict | None,
|
||||
shipped: dict[str, str],
|
||||
shipped: dict[str, str | None],
|
||||
target_cores: set[str] | None,
|
||||
region_drops: set[str] | None = None,
|
||||
report: list[dict] | None = None,
|
||||
@@ -911,6 +920,44 @@ def _mark_hidden(
|
||||
return placed
|
||||
|
||||
|
||||
def _builder_placement(
|
||||
packed_order: list[tuple[str, str, str | None]], case_insensitive: bool,
|
||||
) -> tuple[dict[str, str], dict[str, str | None]]:
|
||||
"""The platform files the builder writes, and every path it keeps.
|
||||
|
||||
The builder walks the systems in name order and skips a path that a
|
||||
written file blocks as a directory, or the reverse: RetroArch declares
|
||||
SGB1.sfc and SGB1.sfc/program.rom, and only the first is in the pack.
|
||||
An entry without a destination (EmuDeck's system:md5 whitelist) is
|
||||
judged but takes no path. Returns (path -> local copy written there,
|
||||
path -> copy or None for each slot kept, a user_provided or external one
|
||||
included).
|
||||
"""
|
||||
from packpaths import _has_path_conflict, _register_path
|
||||
|
||||
shipped: dict[str, str] = {}
|
||||
occupied: dict[str, str | None] = {}
|
||||
seen: set[str] = set()
|
||||
folded: set[str] = set()
|
||||
parents: set[str] = set()
|
||||
for _sys_id, full, local in sorted(packed_order, key=lambda item: item[0]):
|
||||
# EmuDeck's system:md5 entries name no file: the builder writes none.
|
||||
if not full or full.endswith("/"):
|
||||
continue
|
||||
if (
|
||||
full in occupied
|
||||
or (case_insensitive and full.lower() in folded)
|
||||
or _has_path_conflict(full, seen, parents)
|
||||
):
|
||||
continue
|
||||
_register_path(full, seen, parents)
|
||||
folded.add(full.lower())
|
||||
occupied[full] = local
|
||||
if local is not None:
|
||||
shipped[full] = local
|
||||
return shipped, occupied
|
||||
|
||||
|
||||
def _dest_key(file_entry: dict, sys_id: str) -> str:
|
||||
"""The path a declaration occupies, as the builder keys its pack."""
|
||||
dest = sanitize_pack_path(file_entry.get("destination", file_entry.get("name", "")))
|
||||
@@ -1148,7 +1195,10 @@ def verify_platform(
|
||||
verify_systems, db, base_dest, zip_contents
|
||||
)
|
||||
|
||||
shipped: dict[str, str] = {}
|
||||
# (system, full destination, local copy) of each declaration the builder
|
||||
# writes, and None for one whose slot it keeps without a file from the
|
||||
# collection (user_provided, external). Placed below in the builder's order.
|
||||
packed_order: list[tuple[str, str, str | None]] = []
|
||||
shipped_keys: set[str] = set()
|
||||
results_at: dict[str, list[dict]] = {}
|
||||
for sys_id, system in verify_systems.items():
|
||||
@@ -1172,6 +1222,12 @@ def verify_platform(
|
||||
if stored is not None:
|
||||
result = stored
|
||||
local_path = None
|
||||
# The builder keeps the slot: instructions for a user_provided
|
||||
# file, the download for an external one, whether it arrives
|
||||
# or not. No core file goes there.
|
||||
packed_order.append((
|
||||
sys_id, f"{base_dest}/{destination}" if base_dest else destination, None,
|
||||
))
|
||||
elif not reads_file_contents(mode):
|
||||
result = verify_entry_existence(
|
||||
file_entry,
|
||||
@@ -1216,10 +1272,11 @@ def verify_platform(
|
||||
file_entry, local_path, resolve_status, mode, config.get("hash_rule", "")
|
||||
)[1] is None
|
||||
):
|
||||
shipped.setdefault(
|
||||
packed_order.append((
|
||||
sys_id,
|
||||
f"{base_dest}/{destination}" if base_dest else destination,
|
||||
local_path,
|
||||
)
|
||||
))
|
||||
shipped_keys.add(_dest_key(file_entry, sys_id))
|
||||
if _twin_unmet(
|
||||
result, held_by, file_entry, sys_id,
|
||||
@@ -1252,6 +1309,10 @@ def verify_platform(
|
||||
):
|
||||
file_severity[dest] = sev
|
||||
|
||||
shipped, occupied = _builder_placement(
|
||||
packed_order, bool(config.get("case_insensitive_fs", False))
|
||||
)
|
||||
|
||||
# Cross-reference undeclared files, called exactly as the builder calls
|
||||
# it: its declarations, not the enriched ones (a core file the platform
|
||||
# also declares under another name is packed at the core's own path, 21
|
||||
@@ -1283,7 +1344,7 @@ def verify_platform(
|
||||
|
||||
placed_by: dict[str, tuple[str, str, str]] = {}
|
||||
placed = _mark_hidden(
|
||||
config, undeclared, emulators_dir, db, emu_profiles, shipped, target_cores,
|
||||
config, undeclared, emulators_dir, db, emu_profiles, occupied, target_cores,
|
||||
region_drops, report_for_builder,
|
||||
ExtraResolver(
|
||||
db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index,
|
||||
@@ -1363,10 +1424,23 @@ def verify_platform(
|
||||
)
|
||||
total = len(file_status)
|
||||
|
||||
# The paths a pack built from this report holds, as written in the ZIP
|
||||
# (base_destination stripped, as the builder flattens it). The pipeline
|
||||
# compares this set with the archive: equal counts can hide one file
|
||||
# credited here and another written there.
|
||||
from packpaths import _flat
|
||||
|
||||
pack_destinations = sorted(
|
||||
{_flat(dest, base_dest, True) for dest in shipped} | {
|
||||
_flat(dest, base_dest, True) for dest in placed
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"platform": platform,
|
||||
"verification_mode": mode,
|
||||
"total_files": total,
|
||||
"pack_destinations": pack_destinations,
|
||||
"severity_counts": counts,
|
||||
"status_counts": status_counts,
|
||||
"excluded_files": excluded,
|
||||
@@ -1590,8 +1664,11 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
core_missing_req = len(req_not_in_repo) + len(req_hle_not_in_repo)
|
||||
core_missing_opt = len(opt_not_in_repo)
|
||||
|
||||
# Required gaps are always named below, the optional ones on request.
|
||||
listed = "" if verbose or not core_missing_opt else " (--verbose lists them)"
|
||||
print(
|
||||
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, {core_missing_opt} optional missing"
|
||||
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, "
|
||||
f"{core_missing_opt} optional missing{listed}"
|
||||
)
|
||||
if bundled:
|
||||
names = ", ".join(sorted({u["name"] for u in bundled}))
|
||||
@@ -1603,6 +1680,9 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
_print_undeclared_entry(u, "MISSING (required)", verbose)
|
||||
for u in req_hle_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose)
|
||||
if verbose:
|
||||
for u in opt_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (optional)", verbose)
|
||||
|
||||
if game_data:
|
||||
gd_missing = [u for u in game_data if not u["in_repo"]]
|
||||
@@ -1612,7 +1692,11 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
)
|
||||
gd_unplaced = [u for u in game_data if u["in_repo"] and not u.get("in_pack", True)]
|
||||
if gd_missing or gd_packed or gd_unplaced:
|
||||
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing")
|
||||
listed = "" if verbose or not gd_missing else " (--verbose lists them)"
|
||||
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}")
|
||||
if verbose:
|
||||
for u in gd_missing:
|
||||
_print_undeclared_entry(u, "MISSING (game data)", verbose)
|
||||
if gd_unplaced:
|
||||
print(
|
||||
f" Game data held but not packed: {len(gd_unplaced)} "
|
||||
@@ -1951,7 +2035,7 @@ def verify_emulator(
|
||||
|
||||
dest_hint = file_entry.get("path", "")
|
||||
local_path, resolve_status = resolve_local_file(
|
||||
{**file_entry, "source_profile": emu_name},
|
||||
{**with_archive_members(file_entry, files), "source_profile": emu_name},
|
||||
db,
|
||||
zip_contents,
|
||||
dest_hint=dest_hint,
|
||||
@@ -2151,6 +2235,11 @@ def verify_emulator(
|
||||
"emulators": [n for n, _ in selected],
|
||||
"verification_mode": label,
|
||||
"total_files": total,
|
||||
# The paths the emulator pack fills with the profile's files, for the
|
||||
# pipeline's comparison with the pack built in memory.
|
||||
"pack_destinations": sorted(
|
||||
dest for dest, status in file_status.items() if status == Status.OK
|
||||
),
|
||||
"severity_counts": counts,
|
||||
"status_counts": status_counts,
|
||||
"details": details,
|
||||
@@ -2331,6 +2420,7 @@ def _refuse_listing_narrowings(
|
||||
("--json", args.json),
|
||||
("--strict", args.strict),
|
||||
("--verbose", args.verbose),
|
||||
("--destinations", args.destinations),
|
||||
):
|
||||
if on:
|
||||
parser.error(f"{flag} is incompatible with {listing}")
|
||||
@@ -2346,6 +2436,10 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
|
||||
"--target requires --platform or --all"),
|
||||
(args.target and (args.emulator or args.system),
|
||||
"--target is incompatible with --emulator and --system"),
|
||||
(args.destinations and not (args.platform or args.all),
|
||||
"--destinations requires --platform or --all"),
|
||||
(args.destinations and getattr(args, "region", None),
|
||||
"--destinations describes the full pack and is incompatible with --region"),
|
||||
):
|
||||
if refused:
|
||||
parser.error(message)
|
||||
@@ -2423,6 +2517,11 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
help="Show emulator ground truth details",
|
||||
)
|
||||
parser.add_argument("--json", action="store_true", help="JSON output")
|
||||
parser.add_argument(
|
||||
"--destinations", metavar="FILE",
|
||||
help="With --platform or --all, also write the paths each pack holds "
|
||||
"according to this report to FILE (JSON), for comparison with the archive",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strict",
|
||||
action="store_true",
|
||||
@@ -2555,6 +2654,15 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
if len(group_results) > 1:
|
||||
print(_overall_summary(group_results))
|
||||
|
||||
if args.destinations:
|
||||
held = {}
|
||||
for (result, names), (group, _rep) in zip(group_results, groups):
|
||||
for name, key in zip(names, group):
|
||||
held[name] = {
|
||||
"platform": key, "pack_destinations": result["pack_destinations"],
|
||||
}
|
||||
Path(args.destinations).write_text(json.dumps(held, indent=1, sort_keys=True))
|
||||
|
||||
if args.json:
|
||||
for r in all_results.values():
|
||||
r["details"] = _json_details(r["details"], args.verbose)
|
||||
|
||||
@@ -349,5 +349,65 @@ class AnArchiveRankedByWhatItHolds(unittest.TestCase):
|
||||
self.assertIn('entry["contents"] = f["contents"]', report)
|
||||
|
||||
|
||||
class ALooseEntryNamingItsArchive(unittest.TestCase):
|
||||
"""gngeopi declares neogeo.zip loose and, apart, the ROMs it opens inside.
|
||||
The archive pass took the copy holding them, the loose entry resolved by
|
||||
name alone to another copy, and the profile blocked its own archive:
|
||||
verify said missing, the pack said missing, and the ZIP held it."""
|
||||
|
||||
def test_verify_and_the_pack_ship_the_archive_once(self):
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
import yaml
|
||||
|
||||
from common import _emulator_profiles_cache
|
||||
from generate_pack import generate_emulator_pack
|
||||
from verify import verify_emulator
|
||||
|
||||
_emulator_profiles_cache.clear()
|
||||
(REPO_ROOT / "tmp").mkdir(exist_ok=True)
|
||||
root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
|
||||
self.addCleanup(shutil.rmtree, root, True)
|
||||
previous = os.getcwd()
|
||||
os.chdir(root)
|
||||
self.addCleanup(os.chdir, previous)
|
||||
roms = {"bios.rom": b"B" * 32, "lo.lo": b"L" * 32, "rev.rom": b"R" * 32}
|
||||
files, by_name = {}, {}
|
||||
# Two revisions, neither holding the other: index order picks A.
|
||||
for path, members in (("bios/A/set.zip", ["bios.rom", "rev.rom"]),
|
||||
("bios/B/set.zip", ["bios.rom", "lo.lo"])):
|
||||
Path(path).parent.mkdir(parents=True)
|
||||
with zipfile.ZipFile(path, "w") as zf:
|
||||
for member in members:
|
||||
zf.writestr(member, roms[member])
|
||||
sha1 = hashlib.sha1(Path(path).read_bytes()).hexdigest()
|
||||
files[sha1] = {"path": path, "name": "set.zip", "sha1": sha1, "size": 1}
|
||||
by_name.setdefault("set.zip", []).append(sha1)
|
||||
db = {"files": files, "indexes": {"by_name": by_name, "by_md5": {},
|
||||
"by_path_suffix": {}, "by_crc32": {}}}
|
||||
Path("emulators").mkdir()
|
||||
Path("emulators/neo.yml").write_text(yaml.safe_dump({
|
||||
"emulator": "Neo", "type": "libretro", "systems": ["snk-neogeo-mvs"],
|
||||
"files": [{"name": "set.zip", "required": True, "category": "bios_zip"}] + [
|
||||
{"name": name, "archive": "set.zip", "required": True,
|
||||
"crc32": format(zlib.crc32(data), "08x")}
|
||||
for name, data in roms.items() if name != "rev.rom"
|
||||
],
|
||||
}))
|
||||
report = verify_emulator(["neo"], "emulators", db, platforms_dir="platforms")
|
||||
with redirect_stdout(io.StringIO()) as out:
|
||||
zip_path = generate_emulator_pack(
|
||||
["neo"], "emulators", db, "bios", "out", offline=True,
|
||||
platforms_dir="platforms",
|
||||
)
|
||||
with zipfile.ZipFile(zip_path) as zf, zf.open("set.zip") as inner:
|
||||
held = zipfile.ZipFile(io.BytesIO(inner.read())).namelist()
|
||||
self.assertEqual(sorted(held), ["bios.rom", "lo.lo"])
|
||||
self.assertNotIn("missing", out.getvalue())
|
||||
self.assertEqual({d["status"] for d in report["details"]}, {"ok"})
|
||||
self.assertEqual(report["total_files"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -21,6 +21,7 @@ from cross_reference import ( # noqa: E402
|
||||
_build_supplemental_index,
|
||||
cross_reference,
|
||||
entry_source,
|
||||
print_report,
|
||||
)
|
||||
|
||||
|
||||
@@ -38,6 +39,44 @@ class GapReportSources(unittest.TestCase):
|
||||
entry = {"name": "wave1.bin", "sha256": "AB" * 32}
|
||||
self.assertEqual(entry_source(entry, _index(by_sha256={"ab" * 32: "s"})), "bios")
|
||||
|
||||
def test_a_name_never_wins_over_a_declared_hash(self):
|
||||
"""zquestclassic pins ending.mid by sha1, the collection holds only
|
||||
Hydra Castle Labyrinth's ending.mid: the report called it held."""
|
||||
other = "9" * 40
|
||||
index = _index(
|
||||
by_name={"ending.mid": [other]}, by_name_lower={"ending.mid": "ending.mid"},
|
||||
by_path_suffix={"data/midi/ending.mid": [other]},
|
||||
db_files={other: {"path": "bios/hcl/ending.mid", "md5": "1" * 32, "size": 9}},
|
||||
)
|
||||
pinned = {"name": "ending.mid", "sha1": "5" * 40}
|
||||
self.assertIsNone(entry_source(pinned, index))
|
||||
self.assertIsNone(entry_source({**pinned, "path": "data/midi/ending.mid"}, index))
|
||||
self.assertEqual(entry_source({"name": "ending.mid"}, index), "bios")
|
||||
self.assertEqual(entry_source({"name": "ending.mid", "sha1": other}, index), "bios")
|
||||
self.assertEqual(entry_source({"name": "ending.mid", "md5": "1" * 32}, index), "bios")
|
||||
|
||||
def test_the_heading_and_the_total_count_bundled_files_alike(self):
|
||||
"""abuse printed "Gaps: 3 missing" above three [BUNDLED] files and
|
||||
"0 missing (need to be sourced)" below."""
|
||||
import contextlib
|
||||
import io
|
||||
|
||||
report = {"abuse": {
|
||||
"emulator": "Abuse", "systems": [], "gaps": 3, "total_files": 3,
|
||||
"platform_covered": 0, "gap_missing": 3,
|
||||
"gap_details": [
|
||||
{"name": f"f{i}", "required": True, "source": "missing", "bundled": True,
|
||||
"note": ""}
|
||||
for i in range(3)
|
||||
],
|
||||
}}
|
||||
with contextlib.redirect_stdout(io.StringIO()) as out:
|
||||
print_report(report)
|
||||
text = out.getvalue()
|
||||
self.assertIn("Gaps: 3 bundled", text)
|
||||
self.assertNotIn("3 missing", text)
|
||||
self.assertIn("0 missing (need to be sourced)", text)
|
||||
|
||||
def test_a_directory_does_not_stand_for_a_file(self):
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
|
||||
(Path(tmp) / "bios" / "cpcemu" / "ROM").mkdir(parents=True)
|
||||
|
||||
@@ -83,6 +83,40 @@ class ReadersResolve(unittest.TestCase):
|
||||
self.assertNotIn("ioquake3", et)
|
||||
|
||||
|
||||
class ReadersFollowTheDataTrees(unittest.TestCase):
|
||||
"""Dolphin names codehandler.bin without a hash. The pack takes it from
|
||||
data/dolphin-sys, and the site credited Dolphin and Triforce with the
|
||||
Dolphin-MMJR copy, another revision, because the consumers resolved
|
||||
without the data trees. Reads the repository."""
|
||||
|
||||
def test_dolphin_does_not_read_the_mmjr_copy(self):
|
||||
import os
|
||||
|
||||
from common import load_database, load_emulator_profiles
|
||||
|
||||
if not (REPO_ROOT / "database.json").is_file() or not (
|
||||
REPO_ROOT / "data" / "dolphin-sys" / "codehandler.bin"
|
||||
).is_file():
|
||||
self.skipTest("database.json or data/dolphin-sys is absent")
|
||||
previous = os.getcwd()
|
||||
os.chdir(REPO_ROOT)
|
||||
try:
|
||||
db = load_database("database.json")
|
||||
profiles = load_emulator_profiles("emulators")
|
||||
consumers = FileConsumers(db)
|
||||
consumers.add_emulators({"dolphin": profiles["dolphin"]})
|
||||
finally:
|
||||
os.chdir(previous)
|
||||
mmjr = next(
|
||||
(sha1 for sha1, entry in db["files"].items()
|
||||
if entry["path"] == "bios/Other/dolphin-mmjr/codehandler.bin"),
|
||||
None,
|
||||
)
|
||||
if mmjr is None:
|
||||
self.skipTest("the Dolphin-MMJR copy is not collected")
|
||||
self.assertNotIn("dolphin", consumers.emulators.get(mmjr, set()))
|
||||
|
||||
|
||||
def tearDownModule():
|
||||
_TMP.cleanup()
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
from cross_reference import cross_reference # noqa: E402
|
||||
from validation import declaration_settles # noqa: E402
|
||||
from validation import case_folded_slots, declaration_settles # noqa: E402
|
||||
|
||||
SC55 = {"name": "ROM1.BIN", "path": "SC-55/ROM1.BIN", "size": 32768,
|
||||
"validation": ["size"], "required": True}
|
||||
@@ -45,6 +45,33 @@ class OneRule(unittest.TestCase):
|
||||
self.assertIn("declaration_settles(", source, module)
|
||||
|
||||
|
||||
class ACaseFoldingFilesystem(unittest.TestCase):
|
||||
"""RetroBat declares 32X_G_BIOS.BIN, BlastEm opens 32X_G_BIOS.bin at the
|
||||
same place. The exact-case name test reported it undeclared before the
|
||||
slot was ever compared."""
|
||||
|
||||
declared = {"32X_G_BIOS.BIN": [(256, "32x_g_bios.bin")]}
|
||||
entry = {"name": "32X_G_BIOS.bin"}
|
||||
|
||||
def test_the_slot_settles_it_where_case_folds(self):
|
||||
folded = case_folded_slots({"case_insensitive_fs": True}, self.declared)
|
||||
self.assertTrue(
|
||||
declaration_settles(self.entry, "32X_G_BIOS.bin", self.declared, "", folded)
|
||||
)
|
||||
|
||||
def test_two_spellings_are_two_files_elsewhere(self):
|
||||
folded = case_folded_slots({"case_insensitive_fs": False}, self.declared)
|
||||
self.assertFalse(
|
||||
declaration_settles(self.entry, "32X_G_BIOS.bin", self.declared, "", folded)
|
||||
)
|
||||
|
||||
def test_another_place_settles_nothing(self):
|
||||
folded = case_folded_slots({"case_insensitive_fs": True}, self.declared)
|
||||
self.assertFalse(declaration_settles(
|
||||
self.entry, "32X_G_BIOS.bin", self.declared, "blastem/32X_G_BIOS.bin", folded
|
||||
))
|
||||
|
||||
|
||||
class TheRunningBuildPlacesTheGap(unittest.TestCase):
|
||||
"""Batocera runs Cemu standalone, which reads keys.txt at its
|
||||
standalone_path. The gap report gave the libretro Cemu/keys.txt while
|
||||
|
||||
+22
-10
@@ -98,7 +98,7 @@ class TheReportSaysSo(unittest.TestCase):
|
||||
name: verify_platform(
|
||||
load_platform_config(name, "platforms"), db, "emulators", profiles
|
||||
)
|
||||
for name in ("retroarch", "retrobat")
|
||||
for name in ("retroarch", "retrobat", "lakka")
|
||||
}
|
||||
finally:
|
||||
os.chdir(previous)
|
||||
@@ -109,16 +109,28 @@ class TheReportSaysSo(unittest.TestCase):
|
||||
return u
|
||||
self.skipTest(f"{emulator} {path} is not in the cross-reference")
|
||||
|
||||
def test_a_case_twin_with_other_bytes_is_left_out(self):
|
||||
entry = self._entry("NP2kai", "np2kai/bios.rom", "retrobat")
|
||||
self.assertIs(entry.get("in_pack"), False)
|
||||
self.assertIn("another case", entry.get("omitted", ""))
|
||||
def _listed(self, emulator: str, path: str, platform: str) -> bool:
|
||||
return any(
|
||||
u.get("emulator") == emulator and (u.get("path") or u.get("name")) == path
|
||||
for u in self.results[platform]["undeclared_files"]
|
||||
)
|
||||
|
||||
def test_a_twin_with_the_same_bytes_serves_the_core(self):
|
||||
# MesenCE opens bios.col, and RetroArch's pack folds it onto BIOS.col,
|
||||
# the same bytes, which serves the core on a folding filesystem.
|
||||
entry = self._entry("MesenCE", "bios.col")
|
||||
self.assertIsNot(entry.get("in_pack"), False)
|
||||
def test_a_declaration_in_another_case_settles_the_slot(self):
|
||||
"""On a filesystem that folds case the platform's file at the slot is
|
||||
the one the core opens, whatever its spelling, and the slot
|
||||
arbitration judges its bytes. Listed as a core file, RetroBat's
|
||||
32X_G_BIOS.BIN counted BlastEm's 32X_G_BIOS.bin in the pack twice."""
|
||||
for platform, emulator, path in (
|
||||
("retrobat", "NP2kai", "np2kai/bios.rom"),
|
||||
("retrobat", "BlastEm", "32X_G_BIOS.bin"),
|
||||
("retroarch", "MesenCE", "bios.col"),
|
||||
):
|
||||
with self.subTest(platform=platform, path=path):
|
||||
self.assertFalse(self._listed(emulator, path, platform))
|
||||
|
||||
def test_a_filesystem_keeping_both_names_reports_the_other(self):
|
||||
"""Lakka runs on Linux, where bios.col and BIOS.col are two files."""
|
||||
self.assertTrue(self._listed("MesenCE", "bios.col", "lakka"))
|
||||
|
||||
def test_a_directory_under_a_packed_file_is_left_out(self):
|
||||
entry = self._entry("nSide (SFC Balanced)", "SGB1.sfc/lr35902.boot.rom")
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
"""verify and the pack name the same paths, not only the same number of them.
|
||||
|
||||
The pipeline compared counts. RetroArch declares SGB1.sfc and, below it,
|
||||
SGB1.sfc/program.rom: the pack writes the file and skips the path it blocks,
|
||||
verify credited both, and both sides still read 526/526. A user_provided
|
||||
declaration keeps its slot in the pack, while verify let a core file take
|
||||
it and counted one more file from the cores.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
|
||||
class _Fixture(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
from common import _emulator_profiles_cache
|
||||
|
||||
_emulator_profiles_cache.clear()
|
||||
(REPO_ROOT / "tmp").mkdir(exist_ok=True)
|
||||
self.root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
|
||||
self.addCleanup(shutil.rmtree, self.root, True)
|
||||
self.bios = self.root / "bios"
|
||||
self.platforms = self.root / "platforms"
|
||||
self.emulators = self.root / "emulators"
|
||||
for directory in (self.bios, self.platforms, self.emulators):
|
||||
directory.mkdir()
|
||||
files, by_name, by_md5 = {}, {}, {}
|
||||
for name, data in (("sgb.sfc", b"SGB" * 8), ("program.rom", b"PRG" * 8),
|
||||
("core.bin", b"CORE" * 8)):
|
||||
path = self.bios / name
|
||||
path.write_bytes(data)
|
||||
sha1, md5 = hashlib.sha1(data).hexdigest(), hashlib.md5(data).hexdigest()
|
||||
files[sha1] = {"path": str(path), "name": name, "md5": md5, "size": len(data)}
|
||||
by_name[name] = [sha1]
|
||||
by_md5[md5] = sha1
|
||||
self.db = {"files": files, "indexes": {"by_name": by_name, "by_md5": by_md5,
|
||||
"by_crc32": {}, "by_path_suffix": {}}}
|
||||
|
||||
def _platform(self, files: list[dict], cores: list[str] | None = None) -> dict:
|
||||
config = {"platform": "Paths", "verification_mode": "existence",
|
||||
"base_destination": "system", "systems": {"sys": {"files": files}}}
|
||||
if cores is not None:
|
||||
config["cores"] = cores
|
||||
(self.platforms / "paths.yml").write_text(yaml.safe_dump(config))
|
||||
return config
|
||||
|
||||
def _both(self) -> tuple[dict, set[str]]:
|
||||
from common import load_emulator_profiles, load_platform_config
|
||||
from generate_pack import generate_pack
|
||||
from verify import verify_platform
|
||||
|
||||
profiles = load_emulator_profiles(str(self.emulators))
|
||||
report = verify_platform(
|
||||
load_platform_config("paths", str(self.platforms)), self.db,
|
||||
str(self.emulators), profiles,
|
||||
)
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
zip_path = generate_pack(
|
||||
"paths", str(self.platforms), self.db, str(self.bios),
|
||||
str(self.root / "out"), emulators_dir=str(self.emulators),
|
||||
emu_profiles=profiles, offline=True,
|
||||
)
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
members = {n for n in zf.namelist() if n != "README.txt"}
|
||||
return report, members
|
||||
|
||||
|
||||
class APathBlockedByAFile(_Fixture):
|
||||
def test_verify_credits_only_the_file_the_pack_writes(self):
|
||||
self._platform([
|
||||
{"name": "sgb.sfc", "destination": "SGB1.sfc"},
|
||||
{"name": "program.rom", "destination": "SGB1.sfc/program.rom"},
|
||||
])
|
||||
report, members = self._both()
|
||||
self.assertEqual(members, {"SGB1.sfc"})
|
||||
self.assertEqual(set(report["pack_destinations"]), members)
|
||||
|
||||
|
||||
class AUserProvidedSlot(_Fixture):
|
||||
def test_no_core_file_takes_it_on_either_side(self):
|
||||
(self.emulators / "coreemu.yml").write_text(yaml.safe_dump({
|
||||
"emulator": "CoreEmu", "type": "libretro", "systems": ["sys"],
|
||||
"files": [{"name": "core.bin", "required": True}],
|
||||
}))
|
||||
self._platform(
|
||||
[{"name": "manual.bin", "destination": "core.bin", "storage": "user_provided"}],
|
||||
cores=["coreemu"],
|
||||
)
|
||||
report, members = self._both()
|
||||
self.assertEqual(members, {"INSTRUCTIONS_manual.bin.txt"})
|
||||
self.assertEqual(report["packed_extras"], {"bios": 0, "game_data": 0})
|
||||
self.assertEqual(report["pack_destinations"], [])
|
||||
|
||||
|
||||
class AnEntryWithoutAFileTakesNoPath(unittest.TestCase):
|
||||
"""EmuDeck's system:md5 entries name no file. verify credited the bare
|
||||
base destination as a pack path, and the comparison failed on ''."""
|
||||
|
||||
def test_it_is_left_out(self):
|
||||
from verify import _builder_placement
|
||||
|
||||
shipped, occupied = _builder_placement(
|
||||
[("s", "bios/", None), ("s", "", "x"), ("s", "bios/a.bin", "bios/a.bin")], False
|
||||
)
|
||||
self.assertEqual(shipped, {"bios/a.bin": "bios/a.bin"})
|
||||
self.assertEqual(set(occupied), {"bios/a.bin"})
|
||||
|
||||
|
||||
class ThePipelineComparesPaths(unittest.TestCase):
|
||||
"""Equal counts, other paths: the comparison fails and names both sides."""
|
||||
|
||||
def test_a_swap_at_equal_counts_fails(self):
|
||||
import pipeline
|
||||
|
||||
(REPO_ROOT / "tmp").mkdir(exist_ok=True)
|
||||
root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
|
||||
self.addCleanup(shutil.rmtree, root, True)
|
||||
zip_path = root / "Plat_BIOS_Pack.zip"
|
||||
with zipfile.ZipFile(zip_path, "w") as zf:
|
||||
for name in ("a.bin", "b.bin", "README.txt", "INSTRUCTIONS_x.bin.txt"):
|
||||
zf.writestr(name, name)
|
||||
output = f"Generating pack for plat [source=full]...\n {zip_path}: 2 files packed\n"
|
||||
for credited, expected in ((["a.bin", "b.bin"], True), (["a.bin", "c.bin"], False)):
|
||||
with self.subTest(credited=credited):
|
||||
destinations = root / "d.json"
|
||||
destinations.write_text(json.dumps(
|
||||
{"Plat": {"platform": "plat", "pack_destinations": credited}}
|
||||
))
|
||||
previous = os.getcwd()
|
||||
os.chdir(REPO_ROOT)
|
||||
try:
|
||||
with mock_tree_members(), contextlib.redirect_stdout(io.StringIO()) as out:
|
||||
ok = pipeline.check_pack_destinations(str(destinations), output)
|
||||
finally:
|
||||
os.chdir(previous)
|
||||
self.assertIs(ok, expected, out.getvalue())
|
||||
if not expected:
|
||||
self.assertIn("verify only 1: c.bin", out.getvalue())
|
||||
self.assertIn("pack only 1: b.bin", out.getvalue())
|
||||
|
||||
def test_the_run_compares_every_emulator_pack(self):
|
||||
source = (REPO_ROOT / "scripts" / "pipeline.py").read_text(encoding="utf-8")
|
||||
self.assertIn("ok = check_pack_destinations(destinations_file, pack_output) and ok", source)
|
||||
self.assertIn("ok = check_emulator_packs() and ok", source)
|
||||
self.assertIn('"--destinations", destinations_file', source)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def mock_tree_members():
|
||||
import pipeline
|
||||
|
||||
original = pipeline._tree_members
|
||||
pipeline._tree_members = lambda *_args: {}
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
pipeline._tree_members = original
|
||||
|
||||
|
||||
class EveryEmulatorPackMatchesItsReport(unittest.TestCase):
|
||||
"""Reads the repository: every profile, both builds, built in memory."""
|
||||
|
||||
def test_the_paths_agree(self):
|
||||
if not (REPO_ROOT / "database.json").is_file():
|
||||
self.skipTest("database.json is not built")
|
||||
import pipeline
|
||||
|
||||
previous = os.getcwd()
|
||||
os.chdir(REPO_ROOT)
|
||||
try:
|
||||
with contextlib.redirect_stdout(io.StringIO()) as out:
|
||||
ok = pipeline.check_emulator_packs()
|
||||
finally:
|
||||
os.chdir(previous)
|
||||
self.assertTrue(ok, out.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1223,5 +1223,63 @@ class NoMarketStopsAtAustralia(unittest.TestCase):
|
||||
self.assertEqual(tagged, [])
|
||||
|
||||
|
||||
class ARefusedRegionalFileHoldsNoSlot(unittest.TestCase):
|
||||
"""rg_us.bin is held but its md5 contradicts the one the core checks, so
|
||||
the emulator pack refuses it. Counted held, it won the region and the
|
||||
Japanese BIOS was withdrawn: --emulator --region us shipped no BIOS for
|
||||
the system, and verify, reading the same drops, did not see it."""
|
||||
|
||||
def test_the_fallback_stays_in_pack_and_report(self):
|
||||
import hashlib
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
import yaml
|
||||
|
||||
from generate_pack import generate_emulator_pack
|
||||
from verify import verify_emulator
|
||||
|
||||
common._emulator_profiles_cache.clear()
|
||||
repo = Path(__file__).resolve().parents[1]
|
||||
(repo / "tmp").mkdir(exist_ok=True)
|
||||
root = Path(tempfile.mkdtemp(dir=repo / "tmp"))
|
||||
self.addCleanup(shutil.rmtree, root, True)
|
||||
previous = os.getcwd()
|
||||
os.chdir(root)
|
||||
self.addCleanup(os.chdir, previous)
|
||||
files, by_name = {}, {}
|
||||
for name, data in (("rg_us.bin", b"NOT THE US DUMP"), ("rg_jp.bin", b"JP DUMP")):
|
||||
path = Path("bios") / name
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
sha1 = hashlib.sha1(data).hexdigest()
|
||||
files[sha1] = {"path": str(path), "name": name, "size": len(data),
|
||||
"md5": hashlib.md5(data).hexdigest()}
|
||||
by_name[name] = [sha1]
|
||||
db = {"files": files, "indexes": {"by_name": by_name, "by_md5": {},
|
||||
"by_crc32": {}, "by_path_suffix": {}}}
|
||||
Path("emulators").mkdir()
|
||||
Path("emulators/rgemu.yml").write_text(yaml.safe_dump({
|
||||
"emulator": "RgEmu", "type": "libretro", "systems": ["sys"],
|
||||
"files": [
|
||||
{"name": "rg_us.bin", "region": ["north-america"], "required": True,
|
||||
"md5": hashlib.md5(b"THE US DUMP").hexdigest(), "validation": ["md5"]},
|
||||
{"name": "rg_jp.bin", "region": ["japan"], "required": True},
|
||||
],
|
||||
}))
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
zip_path = generate_emulator_pack(
|
||||
["rgemu"], "emulators", db, "bios", "out", regions=["north-america"],
|
||||
offline=True, platforms_dir="platforms",
|
||||
)
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
self.assertIn("rg_jp.bin", zf.namelist())
|
||||
report = verify_emulator(
|
||||
["rgemu"], "emulators", db, regions=["north-america"], platforms_dir="platforms",
|
||||
)
|
||||
self.assertIn("rg_jp.bin", report["pack_destinations"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -349,6 +349,7 @@ class TestArbitration(unittest.TestCase):
|
||||
def _conflict(self):
|
||||
platform = slots.Claim(
|
||||
"platform", "GC/JAP/IPL.bin", "IPL.bin",
|
||||
entry={"md5": "m" * 32, "sha1": "a" * 40},
|
||||
local_path="bios/Console/GC/USA/IPL.bin", status="md5_exact",
|
||||
)
|
||||
profile = slots.Claim(
|
||||
@@ -376,13 +377,75 @@ class TestArbitration(unittest.TestCase):
|
||||
self.assertEqual(decision.winner.origin, "profile")
|
||||
self.assertFalse(decision.serves_both)
|
||||
|
||||
def test_a_hashless_entry_is_a_path_in_an_md5_platform(self):
|
||||
"""EmuDeck declares neocdz.zip with no md5 and checkBIOS.sh never
|
||||
names it: the frontend checks nothing there, so the core's file
|
||||
serves both."""
|
||||
conflict = self._conflict()
|
||||
conflict.platform_claim.entry = {"name": "IPL.bin"}
|
||||
for mode in ("md5", "sha1"):
|
||||
with self.subTest(mode=mode):
|
||||
self.assertTrue(slots.arbitrate(conflict, mode).serves_both)
|
||||
# RomM compares size then any hash: a hash of another kind is content.
|
||||
conflict.platform_claim.entry = {"name": "IPL.bin", "sha1": "a" * 40, "size": 3}
|
||||
self.assertFalse(
|
||||
slots.arbitrate(conflict, "md5", rule="size_and_any_hash").serves_both
|
||||
)
|
||||
conflict.platform_claim.entry = {"name": "x.zip", "zipped_file": "IPL.bin"}
|
||||
self.assertFalse(slots.arbitrate(conflict, "md5").serves_both)
|
||||
|
||||
def test_the_pack_serves_the_proven_file_at_a_hashless_md5_entry(self):
|
||||
config = {
|
||||
"verification_mode": "md5",
|
||||
"cores": ["dolphin"],
|
||||
"systems": {"console": {"files": [
|
||||
{"name": "IPL.bin", "destination": "IPL.bin"},
|
||||
{"name": "IPL.bin", "destination": "hashed/IPL.bin", "md5": "m" * 32},
|
||||
]}},
|
||||
}
|
||||
profiles = {"dolphin": {"type": "libretro", "systems": ["console"], "files": [
|
||||
{"name": "IPL.bin", "md5": "n" * 32},
|
||||
{"name": "IPL.bin", "path": "hashed/IPL.bin", "md5": "n" * 32},
|
||||
]}}
|
||||
overrides = slots.pack_overrides(config, profiles, REGIONS_DB)
|
||||
# The hashed declaration keeps the file the frontend checks.
|
||||
self.assertEqual(overrides, {"IPL.bin": "bios/Console/GC/JAP/IPL.bin"})
|
||||
|
||||
def test_the_reported_line_gives_the_ground_for_the_decision(self):
|
||||
kept = slots.format_decision(slots.arbitrate(self._conflict(), "md5"))
|
||||
self.assertIn("verifies content", kept)
|
||||
# A fact about each layer, no verdict on the platform's list.
|
||||
self.assertIn("the file dolphin reads", kept)
|
||||
self.assertNotIn("wrong", slots.__loader__.get_source("slots").lower())
|
||||
served = slots.format_decision(slots.arbitrate(self._conflict(), "existence"))
|
||||
self.assertIn("both are satisfied", served)
|
||||
|
||||
|
||||
class TheStrongestProofTakesAnUndeclaredPath(unittest.TestCase):
|
||||
"""No platform declares RetroDECK's hiscore.dat. fbalpha2012 asks for it
|
||||
by name, mame2015 pins it by sha1: the first in profile order took the
|
||||
path, whatever the proof, and mame2015 received another revision."""
|
||||
|
||||
def test_a_held_hash_beats_a_bare_name_whatever_the_order(self):
|
||||
from packextras import _collect_emulator_extras
|
||||
|
||||
config = {
|
||||
"platform": "P", "verification_mode": "existence",
|
||||
"cores": ["aaa", "zzz"],
|
||||
"systems": {"console": {"files": [{"name": "other.bin", "destination": "other.bin"}]}},
|
||||
}
|
||||
profiles = {
|
||||
"aaa": {"emulator": "Aaa", "type": "libretro", "systems": ["console"],
|
||||
"files": [{"name": "IPL.bin", "required": True}]},
|
||||
"zzz": {"emulator": "Zzz", "type": "libretro", "systems": ["console"],
|
||||
"files": [{"name": "IPL.bin", "required": True, "md5": "n" * 32}]},
|
||||
}
|
||||
extras = _collect_emulator_extras(config, "", REGIONS_DB, set(), "", profiles)
|
||||
winner = next(e for e in extras if e["destination"] == "IPL.bin")
|
||||
self.assertEqual(winner["source_profile"], "zzz")
|
||||
self.assertEqual({c["source_profile"] for c in winner["other_claims"]}, {"aaa"})
|
||||
|
||||
|
||||
class TestBuilderAndVerifierAgree(unittest.TestCase):
|
||||
"""The two tools must settle a contested destination the same way."""
|
||||
|
||||
|
||||
@@ -148,5 +148,43 @@ class ExistenceReportsWhatThePackReports(unittest.TestCase):
|
||||
self.assertEqual(result["status"], verify.Status.OK)
|
||||
self.assertIn("declared hash 9d2ff233", result["discrepancy"])
|
||||
|
||||
|
||||
class TheDestinationsFlagNeedsAPack(unittest.TestCase):
|
||||
"""--destinations describes a platform's full pack: another mode refuses it."""
|
||||
|
||||
def test_refused_outside_platform_modes(self):
|
||||
for mode in (("--emulator", "handy"), ("--platform", "retroarch", "--region", "us")):
|
||||
with self.subTest(mode=mode):
|
||||
result = _run("scripts/verify.py", *mode, "--destinations", "tmp/never.json")
|
||||
self.assertEqual(result.returncode, 2, result.stdout[-300:])
|
||||
self.assertIn("--destinations", result.stderr)
|
||||
|
||||
|
||||
class EveryGapCountedIsNamed(unittest.TestCase):
|
||||
"""verify --platform batocera --verbose counted 3 optional core files and
|
||||
427 game data files missing and named none of them."""
|
||||
|
||||
def test_verbose_names_them_and_the_summary_says_how(self):
|
||||
import contextlib
|
||||
import io
|
||||
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
import verify # noqa: PLC0415
|
||||
|
||||
result = {"undeclared_files": [
|
||||
{"name": "opt.rom", "emulator": "core", "required": False, "in_repo": False},
|
||||
{"name": "level.dat", "emulator": "engine", "required": True, "in_repo": False,
|
||||
"category": "game_data"},
|
||||
]}
|
||||
with contextlib.redirect_stdout(io.StringIO()) as quiet:
|
||||
verify._print_undeclared_section(result, verbose=False)
|
||||
self.assertIn("1 optional missing (--verbose lists them)", quiet.getvalue())
|
||||
self.assertIn("1 missing (--verbose lists them)", quiet.getvalue())
|
||||
with contextlib.redirect_stdout(io.StringIO()) as verbose:
|
||||
verify._print_undeclared_section(result, verbose=True)
|
||||
self.assertIn("MISSING (optional): core needs opt.rom", verbose.getvalue())
|
||||
self.assertIn("MISSING (game data): engine needs level.dat", verbose.getvalue())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -510,7 +510,7 @@ This executes in sequence:
|
||||
2. `refresh_data_dirs.py` - update data directories (skipped with `--offline`)
|
||||
3. `verify.py --all` - verify all platforms including the new one
|
||||
4. `generate_pack.py --all` - build ZIP packs + install and target manifests
|
||||
5. Consistency check - verify counts match between verify and pack
|
||||
5. Consistency check: verify and the pack agree on counts and on the paths each archive holds
|
||||
6. Pack integrity - extract ZIPs and verify hashes per platform mode
|
||||
7. `generate_readme.py` - regenerate README and CONTRIBUTING
|
||||
8. `generate_site.py` - regenerate documentation site and `mkdocs.yml`
|
||||
@@ -527,7 +527,7 @@ Check the output for:
|
||||
- Pack generation succeeds and includes the expected files
|
||||
- Consistency check passes (verify file counts match pack file counts)
|
||||
|
||||
Verification is not optional. A platform that passes `pipeline.py` today may break tomorrow if upstream changes its data format. Run the full pipeline on every change, even if the modification seems trivial. The consistency check (verify counts must match pack counts) catches subtle issues where files resolve during verification but fail during pack generation, or vice versa.
|
||||
Verification is not optional. A platform that passes `pipeline.py` today may break tomorrow if upstream changes its data format. Run the full pipeline on every change, even if the modification seems trivial. The consistency check (verify and the pack must agree on counts and paths) catches subtle issues where files resolve during verification but fail during pack generation, or vice versa.
|
||||
|
||||
## Checklist
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ Every module in `tests/`, by the area it guards.
|
||||
| Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` |
|
||||
| File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_shipped_content`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` |
|
||||
| Verification, cross-reference and profile checks | synthetic + profiles | `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_cross_reference_sources`, `test_file_consumers`, `test_fileless_audit`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice` |
|
||||
| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_pack_conformance_dirs`, `test_pack_counts`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
|
||||
| Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_pack_conformance_dirs`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
|
||||
| Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
|
||||
| Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
|
||||
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` |
|
||||
|
||||
+8
-4
@@ -42,7 +42,7 @@ Pipeline steps:
|
||||
| 4b | Generate install manifests | `--skip-packs` |
|
||||
| 4c | Generate target manifests | `--skip-packs` |
|
||||
| 4d | Collected copies only an emulator install provides (`bundled_only.py --strict`, gates the run) | - |
|
||||
| 5 | Consistency check | if verify or pack skipped |
|
||||
| 5 | Consistency check: counts, paths per archive, every emulator pack | if verify or pack skipped |
|
||||
| 6 | Pack integrity (extract + hash) | `--skip-packs` |
|
||||
| 7 | Generate README | `--skip-docs` |
|
||||
| 8 | Generate site | `--skip-docs` |
|
||||
@@ -78,9 +78,12 @@ alone, run `python scripts/check_buildbot_system.py`.
|
||||
and manifest names. `--manifest --target switch` once overwrote the
|
||||
manifest every installer downloads with a narrowed one.
|
||||
- **A step that did not run says SKIPPED**, never OK.
|
||||
- **verify.py and generate_pack.py read the same functions.** A count that
|
||||
differs between the report and the pack is a defect, which is what step 5
|
||||
checks.
|
||||
- **verify.py and generate_pack.py read the same functions.** A count or a
|
||||
path that differs between the report and the pack is a defect, which is
|
||||
what step 5 checks: the counts, the paths each archive holds against the
|
||||
ones verify credits (`--destinations`), and every emulator pack, each
|
||||
profile in each build, built in memory against `verify --emulator`.
|
||||
Equal counts once hid four RetroArch paths credited and never written.
|
||||
- **Generators write through `write_if_changed()`.** A new timestamp alone
|
||||
rewrites nothing, so a diff on a generated file always means content.
|
||||
- **One run per output directory.** Pack generation takes a lock on
|
||||
@@ -118,6 +121,7 @@ python scripts/verify.py --emulator handy --strict # exit 1 on a required file
|
||||
python scripts/verify.py --system atari-lynx # single system
|
||||
python scripts/verify.py --platform retroarch --target switch # filter by hardware
|
||||
python scripts/verify.py --platform recalbox --region us # regional priority list
|
||||
python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds
|
||||
python scripts/verify.py --list-emulators # list all emulators
|
||||
python scripts/verify.py --list-systems # list all systems
|
||||
python scripts/verify.py --platform retroarch --list-targets # list available targets
|
||||
|
||||
Reference in new issue
Block a user