mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-10-10 21:43:23 -05:00
Abdessamad Derraz
bf3196ce06
find_undeclared_files decided per entry whether a core requirement can be a gap, through a chain that mixed two skips whose difference is easy to lose. Some record the requirement as settled so no other profile reconsiders it; the rest leave it open, because the same file can be libretro-only in one profile and standalone-only in another and the key carries no emulator. The chain returns a named verdict now, complexity 60 to 45. Twelve tests cover the verdicts and two more cover the distinction end to end: a standalone-only entry seen first must not answer for the profile that needs the file. Collapsing the two skips into one passes every other test in the suite and fails that pair.
1892 lines
68 KiB
Python
1892 lines
68 KiB
Python
#!/usr/bin/env python3
|
|
"""Platform-native BIOS verification engine.
|
|
|
|
Replicates the exact verification logic of each platform:
|
|
- RetroArch/Lakka/RetroPie: file existence only (core_info.c path_is_valid)
|
|
- Batocera: MD5 + checkInsideZip, no required distinction (batocera-systems:1062-1091)
|
|
- Recalbox: MD5 + mandatory/hashMatchMandatory, 3-color severity (Bios.cpp:109-130)
|
|
- RetroBat: same as Batocera
|
|
- EmuDeck: MD5 whitelist per system
|
|
- RetroDECK: MD5 per file via component manifests
|
|
- RomM: size + any hash, no ZIP inspection (firmware.py verify_file_hashes)
|
|
- ROCKNIX: MD5 + checkInsideZip with altmd5 (rocknix-systems checkBios)
|
|
- MiSTer FPGA: MD5 of the file at its destination, no ZIP inspection
|
|
- BizHawk: SHA1 firmware hash verification
|
|
|
|
Cross-references emulator profiles to detect undeclared files used by available cores.
|
|
|
|
Usage:
|
|
python scripts/verify.py --all
|
|
python scripts/verify.py --platform batocera
|
|
python scripts/verify.py --all --include-archived
|
|
python scripts/verify.py --all --json
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
import zipfile
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from common import (
|
|
build_target_cores_cache,
|
|
build_zip_contents_index,
|
|
check_inside_zip,
|
|
compute_hashes,
|
|
expand_platform_declared_names,
|
|
filter_systems_by_target,
|
|
group_identical_platforms,
|
|
list_emulator_profiles,
|
|
list_system_ids,
|
|
load_data_dir_registry,
|
|
load_emulator_profiles,
|
|
load_platform_config,
|
|
md5_composite,
|
|
md5sum,
|
|
require_yaml,
|
|
resolve_local_file,
|
|
resolve_platform_cores,
|
|
sanitize_pack_path,
|
|
)
|
|
|
|
yaml = require_yaml()
|
|
from nativemode import (
|
|
digest_algorithm,
|
|
hash_mismatch_excludes_file,
|
|
normalize as normalize_mode,
|
|
reads_file_contents,
|
|
)
|
|
from validation import (
|
|
_build_validation_index,
|
|
_parse_validation,
|
|
build_ground_truth,
|
|
check_file_validation,
|
|
filter_files_by_mode,
|
|
)
|
|
|
|
DEFAULT_DB = "database.json"
|
|
DEFAULT_PLATFORMS_DIR = "platforms"
|
|
DEFAULT_EMULATORS_DIR = "emulators"
|
|
|
|
|
|
# Status model -aligned with Batocera BiosStatus (batocera-systems:967-969)
|
|
|
|
|
|
class Status:
|
|
OK = "ok"
|
|
UNTESTED = "untested" # file present, hash not confirmed
|
|
MISSING = "missing"
|
|
|
|
|
|
# Severity for per-file required/optional distinction
|
|
class Severity:
|
|
CRITICAL = "critical" # required file missing or bad hash (Recalbox RED)
|
|
WARNING = "warning" # optional missing or hash mismatch (Recalbox YELLOW)
|
|
INFO = "info" # optional missing on existence-only platform
|
|
OK = "ok" # file verified
|
|
|
|
|
|
_STATUS_ORDER = {Status.OK: 0, Status.UNTESTED: 1, Status.MISSING: 2}
|
|
_SEVERITY_ORDER = {
|
|
Severity.OK: 0,
|
|
Severity.INFO: 1,
|
|
Severity.WARNING: 2,
|
|
Severity.CRITICAL: 3,
|
|
}
|
|
|
|
|
|
# Verification functions
|
|
|
|
|
|
def verify_entry_existence(
|
|
file_entry: dict,
|
|
local_path: str | None,
|
|
validation_index: dict[str, dict] | None = None,
|
|
db: dict | None = None,
|
|
) -> dict:
|
|
"""RetroArch verification: path_is_valid() -file exists = OK."""
|
|
name = file_entry.get("name", "")
|
|
required = file_entry.get("required", True)
|
|
if not local_path:
|
|
return {"name": name, "status": Status.MISSING, "required": required}
|
|
result = {"name": name, "status": Status.OK, "required": required}
|
|
if validation_index:
|
|
check = check_file_validation(local_path, name, validation_index)
|
|
if check:
|
|
reason, emus_list = check
|
|
suppressed = False
|
|
if db:
|
|
better = _find_best_variant(
|
|
file_entry, db, local_path, validation_index,
|
|
)
|
|
if better:
|
|
suppressed = True
|
|
if not suppressed:
|
|
emus = ", ".join(emus_list)
|
|
result["discrepancy"] = (
|
|
f"file present (OK) but {emus} says {reason}"
|
|
)
|
|
return result
|
|
|
|
|
|
def verify_entry_md5(
|
|
file_entry: dict,
|
|
local_path: str | None,
|
|
resolve_status: str = "",
|
|
) -> dict:
|
|
"""MD5 verification -Batocera md5sum + Recalbox multi-hash + Md5Composite."""
|
|
name = file_entry.get("name", "")
|
|
expected_md5 = file_entry.get("md5", "")
|
|
zipped_file = file_entry.get("zipped_file")
|
|
required = file_entry.get("required", True)
|
|
base = {"name": name, "required": required}
|
|
|
|
if expected_md5 and "," in expected_md5:
|
|
md5_list = [m.strip().lower() for m in expected_md5.split(",") if m.strip()]
|
|
else:
|
|
md5_list = [expected_md5] if expected_md5 else []
|
|
|
|
if not local_path:
|
|
return {**base, "status": Status.MISSING}
|
|
|
|
if zipped_file:
|
|
found_in_zip = False
|
|
had_error = False
|
|
for md5_candidate in md5_list or [""]:
|
|
result = check_inside_zip(local_path, zipped_file, md5_candidate)
|
|
if result == Status.OK:
|
|
return {**base, "status": Status.OK, "path": local_path}
|
|
if result == "error":
|
|
had_error = True
|
|
elif result != "not_in_zip":
|
|
found_in_zip = True
|
|
if had_error and not found_in_zip:
|
|
return {
|
|
**base,
|
|
"status": Status.UNTESTED,
|
|
"path": local_path,
|
|
"reason": f"{local_path} read error",
|
|
}
|
|
if not found_in_zip:
|
|
return {
|
|
**base,
|
|
"status": Status.UNTESTED,
|
|
"path": local_path,
|
|
"reason": f"{zipped_file} not found inside ZIP",
|
|
}
|
|
return {
|
|
**base,
|
|
"status": Status.UNTESTED,
|
|
"path": local_path,
|
|
"reason": f"{zipped_file} MD5 mismatch inside ZIP",
|
|
}
|
|
|
|
if not md5_list:
|
|
return {**base, "status": Status.OK, "path": local_path}
|
|
|
|
if resolve_status == "md5_exact":
|
|
return {**base, "status": Status.OK, "path": local_path}
|
|
|
|
actual_md5 = md5sum(local_path)
|
|
actual_lower = actual_md5.lower()
|
|
for expected in md5_list:
|
|
if actual_lower == expected.lower():
|
|
return {**base, "status": Status.OK, "path": local_path}
|
|
if len(expected) < 32 and actual_lower.startswith(expected.lower()):
|
|
return {**base, "status": Status.OK, "path": local_path}
|
|
|
|
if ".zip" in os.path.basename(local_path):
|
|
try:
|
|
composite = md5_composite(local_path)
|
|
for expected in md5_list:
|
|
if composite.lower() == expected.lower():
|
|
return {**base, "status": Status.OK, "path": local_path}
|
|
except (zipfile.BadZipFile, OSError):
|
|
pass
|
|
|
|
return {
|
|
**base,
|
|
"status": Status.UNTESTED,
|
|
"path": local_path,
|
|
"reason": f"expected {md5_list[0][:12]}… got {actual_md5[:12]}…",
|
|
}
|
|
|
|
|
|
def verify_entry_sha1(
|
|
file_entry: dict,
|
|
local_path: str | None,
|
|
) -> dict:
|
|
"""SHA1 verification -BizHawk firmware hash check."""
|
|
name = file_entry.get("name", "")
|
|
expected_sha1 = file_entry.get("sha1", "")
|
|
required = file_entry.get("required", True)
|
|
base = {"name": name, "required": required}
|
|
|
|
if not local_path:
|
|
return {**base, "status": Status.MISSING}
|
|
|
|
if not expected_sha1:
|
|
return {**base, "status": Status.OK, "path": local_path}
|
|
|
|
hashes = compute_hashes(local_path)
|
|
actual_sha1 = hashes["sha1"].lower()
|
|
if actual_sha1 == expected_sha1.lower():
|
|
return {**base, "status": Status.OK, "path": local_path}
|
|
|
|
return {
|
|
**base,
|
|
"status": Status.UNTESTED,
|
|
"path": local_path,
|
|
"reason": f"expected {expected_sha1[:12]}… got {actual_sha1[:12]}…",
|
|
}
|
|
|
|
|
|
# Severity mapping per platform
|
|
|
|
|
|
def compute_severity(
|
|
status: str,
|
|
required: bool,
|
|
mode: str,
|
|
hle_fallback: bool = False,
|
|
) -> str:
|
|
"""Map (status, required, verification_mode, hle_fallback) -> severity.
|
|
|
|
Based on native platform behavior + emulator HLE capability:
|
|
- RetroArch (existence): required+missing = warning, optional+missing = info
|
|
- Batocera/Recalbox/RetroBat/EmuDeck (md5): hash-based verification
|
|
- BizHawk (sha1): same severity rules as md5
|
|
- hle_fallback: core works without this file via HLE -> always INFO when missing
|
|
"""
|
|
if status == Status.OK:
|
|
return Severity.OK
|
|
|
|
# HLE fallback: core works without this file regardless of platform requirement
|
|
if hle_fallback and status == Status.MISSING:
|
|
return Severity.INFO
|
|
|
|
if not reads_file_contents(mode):
|
|
if status == Status.MISSING:
|
|
return Severity.WARNING if required else Severity.INFO
|
|
return Severity.OK
|
|
|
|
# md5 mode (Batocera, Recalbox, RetroBat, EmuDeck)
|
|
if status == Status.MISSING:
|
|
return Severity.CRITICAL if required else Severity.WARNING
|
|
if status == Status.UNTESTED:
|
|
return Severity.WARNING
|
|
return Severity.OK
|
|
|
|
|
|
# ZIP content index
|
|
|
|
# Cross-reference: undeclared files used by cores
|
|
|
|
|
|
def _build_expected(file_entry: dict, checks: list[str]) -> dict:
|
|
"""Extract expected validation values from an emulator profile file entry."""
|
|
expected: dict = {}
|
|
if not checks:
|
|
return expected
|
|
if "size" in checks:
|
|
for key in ("size", "min_size", "max_size"):
|
|
if file_entry.get(key) is not None:
|
|
expected[key] = file_entry[key]
|
|
for hash_type in ("crc32", "md5", "sha1", "sha256"):
|
|
if hash_type in checks and file_entry.get(hash_type):
|
|
expected[hash_type] = file_entry[hash_type]
|
|
adler_val = file_entry.get("known_hash_adler32") or file_entry.get("adler32")
|
|
if adler_val:
|
|
expected["adler32"] = adler_val
|
|
return expected
|
|
|
|
|
|
def _name_in_index(
|
|
name: str,
|
|
by_name: dict,
|
|
by_path_suffix: dict | None = None,
|
|
data_names: set[str] | None = None,
|
|
by_name_lower: dict[str, str] | None = None,
|
|
) -> bool:
|
|
"""Check if a name is resolvable in the database indexes or data directories."""
|
|
# Strip trailing slash for directory-type entries (e.g. nestopia/samples/foo/)
|
|
name = name.rstrip("/")
|
|
if name in by_name:
|
|
return True
|
|
basename = name.rsplit("/", 1)[-1] if "/" in name else name
|
|
if basename != name and basename in by_name:
|
|
return True
|
|
# Case-insensitive by_name lookup
|
|
if by_name_lower:
|
|
key = name.lower()
|
|
if key in by_name_lower:
|
|
return True
|
|
if basename != name and basename.lower() in by_name_lower:
|
|
return True
|
|
if by_path_suffix and name in by_path_suffix:
|
|
return True
|
|
if data_names:
|
|
if name in data_names or name.lower() in data_names:
|
|
return True
|
|
if basename != name and (
|
|
basename in data_names or basename.lower() in data_names
|
|
):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _candidate_verdict(
|
|
file_entry: dict,
|
|
fname: str,
|
|
is_standalone: bool,
|
|
include_all: bool,
|
|
declared_names: set,
|
|
) -> str:
|
|
"""Whether a profile entry can be a gap, and whether it is settled.
|
|
|
|
Three answers. "keep" means it is a candidate. "settled" means something
|
|
has answered for it -the platform declares it, or the profile documents
|
|
it as unsourceable -so the same requirement reached from another profile
|
|
must not be reconsidered. "skip" means it does not apply in this context,
|
|
which leaves it open for a profile where it does: the same file can be
|
|
libretro-only here and standalone-only there.
|
|
"""
|
|
if file_entry.get("unsourceable"):
|
|
return "settled"
|
|
# Placeholders stand for a family of files, not a file.
|
|
if "<" in fname or ">" in fname or "*" in fname:
|
|
return "skip"
|
|
# An explicit null path means the user imports it through the UI.
|
|
if "path" in file_entry and file_entry["path"] is None:
|
|
return "skip"
|
|
file_mode = file_entry.get("mode")
|
|
if file_mode == "standalone" and not is_standalone:
|
|
return "skip"
|
|
if file_mode == "libretro" and is_standalone:
|
|
return "skip"
|
|
# Read from somewhere other than the system directory: not a BIOS gap.
|
|
load_from = file_entry.get("load_from", "")
|
|
if load_from and load_from != "system_dir":
|
|
return "skip"
|
|
# Filename-agnostic entries are answered by the builder's own scan.
|
|
if file_entry.get("agnostic"):
|
|
return "skip"
|
|
if not include_all:
|
|
archive = file_entry.get("archive")
|
|
if fname in declared_names or (archive and archive in declared_names):
|
|
return "settled"
|
|
return "keep"
|
|
|
|
|
|
def find_undeclared_files(
|
|
config: dict,
|
|
emulators_dir: str,
|
|
db: dict,
|
|
emu_profiles: dict | None = None,
|
|
target_cores: set[str] | None = None,
|
|
data_names: set[str] | None = None,
|
|
include_all: bool = False,
|
|
declared_names: set[str] | None = None,
|
|
) -> list[dict]:
|
|
"""Find files needed by cores but not declared in platform config.
|
|
|
|
declared_names overrides the default enriched set from
|
|
expand_platform_declared_names. Pass a strict set (YAML names only)
|
|
when building packs so alias-only names still get packed.
|
|
"""
|
|
if declared_names is None:
|
|
declared_names = expand_platform_declared_names(config, db)
|
|
|
|
# Whether the builder drops a file whose local copy contradicts its
|
|
# declared hash, which decides if such a copy counts as held here.
|
|
shipped_on_mismatch = hash_mismatch_excludes_file(
|
|
config.get("verification_mode")
|
|
)
|
|
|
|
# Collect data_directory refs
|
|
declared_dd: set[str] = set()
|
|
for sys_id, system in config.get("systems", {}).items():
|
|
for dd in system.get("data_directories", []):
|
|
ref = dd.get("ref", "")
|
|
if ref:
|
|
declared_dd.add(ref)
|
|
|
|
by_name = db.get("indexes", {}).get("by_name", {})
|
|
by_name_lower = {k.lower(): k for k in by_name}
|
|
by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
|
|
profiles = (
|
|
emu_profiles
|
|
if emu_profiles is not None
|
|
else load_emulator_profiles(emulators_dir)
|
|
)
|
|
|
|
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
|
|
standalone_set = set(str(c) for c in config.get("standalone_cores", []))
|
|
undeclared = []
|
|
seen_files: set[tuple] = set()
|
|
# Track archives: archive_name -> {in_repo, emulator, files: [...], ...}
|
|
archive_entries: dict[tuple, dict] = {}
|
|
|
|
for emu_name, profile in sorted(profiles.items()):
|
|
if profile.get("type") in ("launcher", "alias"):
|
|
continue
|
|
if emu_name not in relevant:
|
|
continue
|
|
|
|
# Skip agnostic profiles entirely (filename-agnostic BIOS detection)
|
|
if profile.get("bios_mode") == "agnostic":
|
|
continue
|
|
|
|
# Check if this profile is standalone: match profile name or any cores: alias
|
|
is_standalone = emu_name in standalone_set or bool(
|
|
standalone_set & {str(c) for c in profile.get("cores", [])}
|
|
)
|
|
|
|
for f in profile.get("files", []):
|
|
fname = f.get("name", "")
|
|
effective_path = (
|
|
f.get("standalone_path") if is_standalone else f.get("path")
|
|
) or fname
|
|
raw_regions = f.get("region") or []
|
|
region_key = tuple(
|
|
str(value) for value in (
|
|
raw_regions if isinstance(raw_regions, list) else [raw_regions]
|
|
)
|
|
)
|
|
# Same-name requirements at different paths, for different systems,
|
|
# or in distinct variant groups are not interchangeable.
|
|
seen_key = (
|
|
fname,
|
|
f.get("archive"),
|
|
effective_path,
|
|
f.get("system"),
|
|
f.get("variant_group"),
|
|
region_key,
|
|
)
|
|
if not fname or seen_key in seen_files:
|
|
continue
|
|
verdict = _candidate_verdict(
|
|
f, fname, is_standalone, include_all, declared_names
|
|
)
|
|
if verdict == "settled":
|
|
seen_files.add(seen_key)
|
|
continue
|
|
if verdict == "skip":
|
|
continue
|
|
|
|
archive = f.get("archive")
|
|
seen_files.add(seen_key)
|
|
|
|
# Archived files are grouped by archive
|
|
if archive:
|
|
archive_key = (
|
|
archive,
|
|
f.get("system"),
|
|
f.get("variant_group"),
|
|
region_key,
|
|
is_standalone,
|
|
)
|
|
if archive_key not in archive_entries:
|
|
in_repo = _name_in_index(
|
|
archive, by_name, by_path_suffix, data_names,
|
|
by_name_lower,
|
|
)
|
|
archive_entries[archive_key] = {
|
|
"profile": emu_name,
|
|
"emulator": profile.get("emulator", emu_name),
|
|
"systems": list(profile.get("systems", [])),
|
|
"system": f.get("system"),
|
|
"region": f.get("region"),
|
|
"variant_group": f.get("variant_group"),
|
|
"name": archive,
|
|
"archive": archive,
|
|
"path": archive,
|
|
"required": False,
|
|
"hle_fallback": False,
|
|
"category": f.get("category", "bios"),
|
|
"in_repo": in_repo,
|
|
"note": "",
|
|
"checks": [],
|
|
"source_ref": None,
|
|
"expected": {},
|
|
"archive_file_count": 0,
|
|
"archive_required_count": 0,
|
|
}
|
|
entry = archive_entries[archive_key]
|
|
entry["archive_file_count"] += 1
|
|
if f.get("required", False):
|
|
entry["archive_required_count"] += 1
|
|
entry["required"] = True
|
|
continue
|
|
|
|
# Determine destination path based on mode
|
|
if is_standalone:
|
|
dest = f.get("standalone_path") or f.get("path") or fname
|
|
else:
|
|
dest = f.get("path") or fname
|
|
|
|
# Resolution: storage flag, then name, then path basename
|
|
storage = f.get("storage", "")
|
|
if storage in ("release", "large_file"):
|
|
in_repo = True
|
|
elif shipped_on_mismatch and (f.get("md5") or f.get("sha1")):
|
|
# The entry states what its content should be and the builder
|
|
# drops a copy that contradicts it, so content decides here
|
|
# too. A name match would not do: generic names collide across
|
|
# systems, and answering yes on one describes a pack that will
|
|
# not contain the file.
|
|
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
|
|
in_repo = _lp is not None and _st != "hash_mismatch"
|
|
else:
|
|
in_repo = _name_in_index(
|
|
fname, by_name, by_path_suffix, data_names, by_name_lower,
|
|
)
|
|
if not in_repo and dest != fname:
|
|
path_base = dest.rsplit("/", 1)[-1]
|
|
in_repo = _name_in_index(
|
|
path_base, by_name, by_path_suffix, data_names,
|
|
by_name_lower,
|
|
)
|
|
if not in_repo:
|
|
# Hash fallback: the repo may hold the content under a
|
|
# different filename (exos21.rom vs exos21.bin).
|
|
#
|
|
# A copy contradicting the declared hash counts as held
|
|
# only where the builder would ship it. Under existence
|
|
# the frontend never opens the file, so the pack carries
|
|
# it and reports the divergence; under a digest mode the
|
|
# frontend would reject it, the builder leaves it out, and
|
|
# calling it available here would describe a pack that
|
|
# does not contain it.
|
|
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
|
|
in_repo = _st != "not_found" and _lp is not None
|
|
if in_repo and _st == "hash_mismatch" and shipped_on_mismatch:
|
|
in_repo = False
|
|
|
|
checks = _parse_validation(f.get("validation"))
|
|
undeclared.append(
|
|
{
|
|
"profile": emu_name,
|
|
"emulator": profile.get("emulator", emu_name),
|
|
"systems": list(profile.get("systems", [])),
|
|
"system": f.get("system"),
|
|
"region": f.get("region"),
|
|
"variant_group": f.get("variant_group"),
|
|
"name": fname,
|
|
"path": dest,
|
|
"required": f.get("required", False),
|
|
"hle_fallback": f.get("hle_fallback", False),
|
|
"category": f.get("category", "bios"),
|
|
"in_repo": in_repo,
|
|
"note": f.get("note", ""),
|
|
"checks": sorted(checks) if checks else [],
|
|
"source_ref": f.get("source_ref"),
|
|
"expected": _build_expected(f, checks),
|
|
"sha1": f.get("sha1"),
|
|
"md5": f.get("md5"),
|
|
}
|
|
)
|
|
|
|
# Append grouped archive entries
|
|
for entry in sorted(archive_entries.values(), key=lambda e: e["name"]):
|
|
undeclared.append(entry)
|
|
|
|
return undeclared
|
|
|
|
|
|
def find_exclusion_notes(
|
|
config: dict,
|
|
emulators_dir: str,
|
|
emu_profiles: dict | None = None,
|
|
target_cores: set[str] | None = None,
|
|
) -> list[dict]:
|
|
"""Document why certain emulator files are intentionally excluded.
|
|
|
|
Reports:
|
|
- Launchers (BIOS managed by standalone emulator)
|
|
- Standalone-only files (not needed in libretro mode)
|
|
- Frozen snapshots with files: [] (code doesn't load .info firmware)
|
|
- Files covered by data_directories
|
|
"""
|
|
profiles = (
|
|
emu_profiles
|
|
if emu_profiles is not None
|
|
else load_emulator_profiles(emulators_dir)
|
|
)
|
|
platform_systems = set()
|
|
for sys_id in config.get("systems", {}):
|
|
platform_systems.add(sys_id)
|
|
|
|
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
|
|
notes = []
|
|
for emu_name, profile in sorted(profiles.items()):
|
|
emu_systems = set(profile.get("systems", []))
|
|
# Match by core resolution OR system intersection (documents all potential emulators)
|
|
if emu_name not in relevant and not (emu_systems & platform_systems):
|
|
continue
|
|
|
|
emu_display = profile.get("emulator", emu_name)
|
|
|
|
# Launcher excluded entirely
|
|
if profile.get("type") == "launcher":
|
|
notes.append(
|
|
{
|
|
"emulator": emu_display,
|
|
"reason": "launcher",
|
|
"detail": profile.get(
|
|
"exclusion_note", "BIOS managed by standalone emulator"
|
|
),
|
|
}
|
|
)
|
|
continue
|
|
|
|
# Profile-level exclusion note (frozen snapshots, etc.)
|
|
exclusion_note = profile.get("exclusion_note")
|
|
if exclusion_note:
|
|
notes.append(
|
|
{
|
|
"emulator": emu_display,
|
|
"reason": "exclusion_note",
|
|
"detail": exclusion_note,
|
|
}
|
|
)
|
|
continue
|
|
|
|
# Count standalone-only files -but only report as excluded if the
|
|
# platform does NOT use this emulator in standalone mode
|
|
standalone_set = set(str(c) for c in config.get("standalone_cores", []))
|
|
is_standalone = emu_name in standalone_set or bool(
|
|
standalone_set & {str(c) for c in profile.get("cores", [])}
|
|
)
|
|
if not is_standalone:
|
|
standalone_files = [
|
|
f for f in profile.get("files", []) if f.get("mode") == "standalone"
|
|
]
|
|
if standalone_files:
|
|
names = [f["name"] for f in standalone_files[:3]]
|
|
more = (
|
|
f" +{len(standalone_files) - 3}"
|
|
if len(standalone_files) > 3
|
|
else ""
|
|
)
|
|
notes.append(
|
|
{
|
|
"emulator": emu_display,
|
|
"reason": "standalone_only",
|
|
"detail": f"{len(standalone_files)} files for standalone mode only ({', '.join(names)}{more})",
|
|
}
|
|
)
|
|
|
|
return notes
|
|
|
|
|
|
# Platform verification
|
|
|
|
|
|
def _find_best_variant(
|
|
file_entry: dict,
|
|
db: dict,
|
|
current_path: str,
|
|
validation_index: dict,
|
|
) -> str | None:
|
|
"""Search for a repo file that passes emulator validation.
|
|
|
|
Two-pass search:
|
|
1. Hash lookup, using the emulator's expected hashes (sha1, md5, sha256,
|
|
crc32) to find candidates directly in the DB indexes. This finds
|
|
variants stored under different filenames (e.g. megacd2_v200_eu.bin
|
|
for bios_CD_E.bin).
|
|
2. Name lookup, checking all files sharing the same name (aliases,
|
|
.variants/ with name-based suffixes).
|
|
|
|
If any candidate on disk passes ``check_file_validation``, the
|
|
discrepancy is suppressed: the repo has what the emulator needs.
|
|
"""
|
|
fname = file_entry.get("name", "")
|
|
if not fname or fname not in validation_index:
|
|
return None
|
|
|
|
files_db = db.get("files", {})
|
|
current_real = os.path.realpath(current_path)
|
|
seen_paths: set[str] = set()
|
|
|
|
def _try_candidate(sha1: str) -> str | None:
|
|
candidate = files_db.get(sha1, {})
|
|
path = candidate.get("path", "")
|
|
if not path or not os.path.exists(path):
|
|
return None
|
|
rp = os.path.realpath(path)
|
|
if rp == current_real or rp in seen_paths:
|
|
return None
|
|
seen_paths.add(rp)
|
|
if check_file_validation(path, fname, validation_index) is None:
|
|
return path
|
|
return None
|
|
|
|
# Pass 1: hash-based lookup from emulator expected values
|
|
ventry = validation_index[fname]
|
|
indexes = db.get("indexes", {})
|
|
for hash_type, db_index_key in (
|
|
("sha1", None),
|
|
("md5", "by_md5"),
|
|
("crc32", "by_crc32"),
|
|
("sha256", "by_sha256"),
|
|
):
|
|
expected = ventry.get(hash_type)
|
|
if not expected:
|
|
continue
|
|
if db_index_key is None:
|
|
# SHA1 is the primary key of files_db
|
|
for h in expected:
|
|
if h in files_db:
|
|
result = _try_candidate(h)
|
|
if result:
|
|
return result
|
|
continue
|
|
db_index = indexes.get(db_index_key, {})
|
|
for h in expected:
|
|
entries = db_index.get(h)
|
|
if not entries:
|
|
continue
|
|
if isinstance(entries, list):
|
|
for sha1 in entries:
|
|
result = _try_candidate(sha1)
|
|
if result:
|
|
return result
|
|
elif isinstance(entries, str):
|
|
result = _try_candidate(entries)
|
|
if result:
|
|
return result
|
|
|
|
# Pass 2: name-based lookup (aliases, .variants/ with same filename)
|
|
by_name = db.get("indexes", {}).get("by_name", {})
|
|
for sha1 in by_name.get(fname, []):
|
|
result = _try_candidate(sha1)
|
|
if result:
|
|
return result
|
|
|
|
return None
|
|
|
|
|
|
def verify_platform(
|
|
config: dict,
|
|
db: dict,
|
|
emulators_dir: str = DEFAULT_EMULATORS_DIR,
|
|
emu_profiles: dict | None = None,
|
|
target_cores: set[str] | None = None,
|
|
data_dir_registry: dict | None = None,
|
|
supplemental_names: set[str] | None = None,
|
|
regions: list[str] | None = None,
|
|
) -> dict:
|
|
"""Verify all BIOS files for a platform, including cross-reference gaps.
|
|
|
|
A region priority list narrows the report to the files a pack built with the
|
|
same list would carry, using the same selection function as the builder.
|
|
"""
|
|
# Normalized once: a typo in a platform YAML must not verify with one
|
|
# mode and be scored with another.
|
|
mode = normalize_mode(config.get("verification_mode"))
|
|
platform = config.get("platform", "unknown")
|
|
|
|
has_zipped = any(
|
|
fe.get("zipped_file")
|
|
for sys in config.get("systems", {}).values()
|
|
for fe in sys.get("files", [])
|
|
)
|
|
zip_contents = build_zip_contents_index(db) if has_zipped else {}
|
|
|
|
# Build HLE + validation indexes from emulator profiles
|
|
profiles = (
|
|
emu_profiles
|
|
if emu_profiles is not None
|
|
else load_emulator_profiles(emulators_dir)
|
|
)
|
|
hle_index: dict[str, bool] = {}
|
|
for profile in profiles.values():
|
|
for f in profile.get("files", []):
|
|
if f.get("hle_fallback"):
|
|
hle_index[f.get("name", "")] = True
|
|
validation_index = _build_validation_index(profiles)
|
|
|
|
# Filter systems by target
|
|
plat_cores = resolve_platform_cores(config, profiles) if target_cores else None
|
|
verify_systems = filter_systems_by_target(
|
|
config.get("systems", {}),
|
|
profiles,
|
|
target_cores,
|
|
platform_cores=plat_cores,
|
|
)
|
|
|
|
# Per-entry results
|
|
details = []
|
|
# Per-destination aggregation
|
|
file_status: dict[str, str] = {}
|
|
file_required: dict[str, bool] = {}
|
|
file_severity: dict[str, str] = {}
|
|
|
|
region_drops: set[str] = set()
|
|
region_extra_dests: dict[tuple[str, str, str], str] = {}
|
|
if regions:
|
|
import region as region_mod
|
|
|
|
# The builder owns pack composition, so it owns the grouping the region
|
|
# pass reads. Grouping the platform files here and the core extras
|
|
# there let the two answer differently on one request: the report kept
|
|
# every core extra a region run withdraws from the pack.
|
|
from generate_pack import platform_region_groups
|
|
|
|
region_groups, region_extra_dests = platform_region_groups(
|
|
config,
|
|
verify_systems,
|
|
emulators_dir,
|
|
db,
|
|
config.get("base_destination", ""),
|
|
profiles,
|
|
target_cores=target_cores,
|
|
)
|
|
region_drops = region_mod.resolve_region_drops(
|
|
region_groups, region_mod.build_region_index(profiles), regions
|
|
)
|
|
|
|
for sys_id, system in verify_systems.items():
|
|
for file_entry in system.get("files", []):
|
|
if region_drops and (
|
|
sanitize_pack_path(
|
|
file_entry.get("destination", file_entry.get("name", ""))
|
|
)
|
|
in region_drops
|
|
):
|
|
continue
|
|
local_path, resolve_status = resolve_local_file(
|
|
file_entry,
|
|
db,
|
|
zip_contents,
|
|
data_dir_registry=data_dir_registry,
|
|
)
|
|
if not reads_file_contents(mode):
|
|
result = verify_entry_existence(
|
|
file_entry,
|
|
local_path,
|
|
validation_index,
|
|
db,
|
|
)
|
|
elif digest_algorithm(mode) == "sha1":
|
|
result = verify_entry_sha1(file_entry, local_path)
|
|
else:
|
|
result = verify_entry_md5(file_entry, local_path, resolve_status)
|
|
# Emulator-level validation: informational for platform packs.
|
|
# Platform verification (MD5) is the authority. Emulator
|
|
# mismatches are reported as discrepancies, not failures.
|
|
if result["status"] == Status.OK and local_path and validation_index:
|
|
fname = file_entry.get("name", "")
|
|
check = check_file_validation(
|
|
local_path, fname, validation_index,
|
|
)
|
|
if check:
|
|
reason, emus_list = check
|
|
better = _find_best_variant(
|
|
file_entry,
|
|
db,
|
|
local_path,
|
|
validation_index,
|
|
)
|
|
if not better:
|
|
emus = ", ".join(emus_list)
|
|
result["discrepancy"] = (
|
|
f"{platform} says OK but {emus} says {reason}"
|
|
)
|
|
result["system"] = sys_id
|
|
result["hle_fallback"] = hle_index.get(file_entry.get("name", ""), False)
|
|
result["ground_truth"] = build_ground_truth(
|
|
file_entry.get("name", ""),
|
|
validation_index,
|
|
)
|
|
details.append(result)
|
|
|
|
# Aggregate by destination
|
|
dest = file_entry.get("destination", file_entry.get("name", ""))
|
|
if not dest:
|
|
dest = f"{sys_id}/{file_entry.get('name', '')}"
|
|
required = file_entry.get("required", True)
|
|
cur = result["status"]
|
|
prev = file_status.get(dest)
|
|
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
|
|
file_status[dest] = cur
|
|
file_required[dest] = required
|
|
hle = hle_index.get(file_entry.get("name", ""), False)
|
|
sev = compute_severity(cur, required, mode, hle)
|
|
prev_sev = file_severity.get(dest)
|
|
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(
|
|
prev_sev, 0
|
|
):
|
|
file_severity[dest] = sev
|
|
|
|
# Count by severity
|
|
counts = {
|
|
Severity.OK: 0,
|
|
Severity.INFO: 0,
|
|
Severity.WARNING: 0,
|
|
Severity.CRITICAL: 0,
|
|
}
|
|
for s in file_severity.values():
|
|
counts[s] = counts.get(s, 0) + 1
|
|
|
|
# Count by file status (ok/untested/missing)
|
|
status_counts: dict[str, int] = {}
|
|
for s in file_status.values():
|
|
status_counts[s] = status_counts.get(s, 0) + 1
|
|
|
|
# Cross-reference undeclared files
|
|
if supplemental_names is None:
|
|
from cross_reference import _build_supplemental_index
|
|
|
|
supplemental_names = _build_supplemental_index()
|
|
undeclared = find_undeclared_files(
|
|
config,
|
|
emulators_dir,
|
|
db,
|
|
emu_profiles,
|
|
target_cores=target_cores,
|
|
data_names=supplemental_names,
|
|
)
|
|
if region_drops:
|
|
undeclared = [
|
|
u
|
|
for u in undeclared
|
|
if region_extra_dests.get(
|
|
(u.get("emulator", ""), u.get("name", ""), u.get("path") or "")
|
|
)
|
|
not in region_drops
|
|
]
|
|
exclusions = find_exclusion_notes(
|
|
config, emulators_dir, emu_profiles, target_cores=target_cores
|
|
)
|
|
|
|
# Ground truth coverage
|
|
gt_filenames = set(validation_index)
|
|
profiled_names: set[str] = set()
|
|
for profile in profiles.values():
|
|
if profile.get("type") in ("launcher", "alias"):
|
|
continue
|
|
for f in profile.get("files", []):
|
|
name = f.get("name", "")
|
|
if name:
|
|
profiled_names.add(name)
|
|
profiled_names.update(f.get("aliases") or [])
|
|
dest_to_name: dict[str, str] = {}
|
|
for sys_id, system in verify_systems.items():
|
|
for fe in system.get("files", []):
|
|
dest = fe.get("destination", fe.get("name", ""))
|
|
if not dest:
|
|
dest = f"{sys_id}/{fe.get('name', '')}"
|
|
dest_to_name.setdefault(dest, fe.get("name", ""))
|
|
with_validation = sum(
|
|
1 for dest in file_status if dest_to_name.get(dest, "") in gt_filenames
|
|
)
|
|
with_profile = sum(
|
|
1 for dest in file_status if dest_to_name.get(dest, "") in profiled_names
|
|
)
|
|
total = len(file_status)
|
|
|
|
return {
|
|
"platform": platform,
|
|
"verification_mode": mode,
|
|
"total_files": total,
|
|
"severity_counts": counts,
|
|
"status_counts": status_counts,
|
|
"undeclared_files": undeclared,
|
|
"exclusion_notes": exclusions,
|
|
"details": details,
|
|
"ground_truth_coverage": {
|
|
"with_validation": with_validation,
|
|
"with_profile": with_profile,
|
|
"platform_only": total - with_validation,
|
|
"total": total,
|
|
"applicable": bool(resolve_platform_cores(config, profiles)),
|
|
},
|
|
}
|
|
|
|
|
|
# Output
|
|
|
|
|
|
def _format_ground_truth_aggregate(ground_truth: list[dict]) -> str:
|
|
"""Format ground truth as a single aggregated line.
|
|
|
|
Example: beetle_psx [md5], pcsx_rearmed [existence]
|
|
"""
|
|
parts = []
|
|
for gt in ground_truth:
|
|
checks_label = "+".join(gt["checks"]) if gt["checks"] else "existence"
|
|
parts.append(f"{gt['emulator']} [{checks_label}]")
|
|
return ", ".join(parts)
|
|
|
|
|
|
def _format_ground_truth_verbose(ground_truth: list[dict]) -> list[str]:
|
|
"""Format ground truth as one line per core with expected values and source ref.
|
|
|
|
Example: handy validates size=512,crc32=0d973c9d [rom.h:48-49]
|
|
"""
|
|
lines = []
|
|
for gt in ground_truth:
|
|
checks_label = "+".join(gt["checks"]) if gt["checks"] else "existence"
|
|
expected = gt.get("expected", {})
|
|
if expected:
|
|
vals = ",".join(f"{k}={v}" for k, v in sorted(expected.items()))
|
|
part = f"{gt['emulator']} validates {vals}"
|
|
else:
|
|
part = f"{gt['emulator']} validates {checks_label}"
|
|
if gt.get("source_ref"):
|
|
part += f" [{gt['source_ref']}]"
|
|
lines.append(part)
|
|
return lines
|
|
|
|
|
|
def _print_ground_truth(gt: list[dict], verbose: bool) -> None:
|
|
"""Print ground truth lines for a file entry."""
|
|
if not gt:
|
|
return
|
|
if verbose:
|
|
for line in _format_ground_truth_verbose(gt):
|
|
print(f" {line}")
|
|
else:
|
|
print(f" {_format_ground_truth_aggregate(gt)}")
|
|
|
|
|
|
def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) -> None:
|
|
"""Print UNTESTED, MISSING, and DISCREPANCY entries from verification details."""
|
|
for d in details:
|
|
if d["status"] == Status.UNTESTED:
|
|
key = f"{d['system']}/{d['name']}"
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
req = "required" if d.get("required", True) else "optional"
|
|
hle = ", HLE available" if d.get("hle_fallback") else ""
|
|
reason = d.get("reason", "")
|
|
print(f" UNTESTED ({req}{hle}): {key} -{reason}")
|
|
_print_ground_truth(d.get("ground_truth", []), verbose)
|
|
for d in details:
|
|
if d["status"] == Status.MISSING:
|
|
key = f"{d['system']}/{d['name']}"
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
req = "required" if d.get("required", True) else "optional"
|
|
hle = ", HLE available" if d.get("hle_fallback") else ""
|
|
print(f" MISSING ({req}{hle}): {key}")
|
|
_print_ground_truth(d.get("ground_truth", []), verbose)
|
|
for d in details:
|
|
disc = d.get("discrepancy")
|
|
if disc:
|
|
key = f"{d['system']}/{d['name']}"
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
print(f" DISCREPANCY: {key} -{disc}")
|
|
_print_ground_truth(d.get("ground_truth", []), verbose)
|
|
|
|
if verbose:
|
|
for d in details:
|
|
if d["status"] == Status.OK:
|
|
key = f"{d['system']}/{d['name']}"
|
|
if key in seen:
|
|
continue
|
|
seen.add(key)
|
|
gt = d.get("ground_truth", [])
|
|
if gt:
|
|
req = "required" if d.get("required", True) else "optional"
|
|
print(f" OK ({req}): {key}")
|
|
for line in _format_ground_truth_verbose(gt):
|
|
print(f" {line}")
|
|
|
|
|
|
def _print_undeclared_entry(u: dict, prefix: str, verbose: bool) -> None:
|
|
"""Print a single undeclared file entry with its validation checks."""
|
|
arc_count = u.get("archive_file_count")
|
|
if arc_count:
|
|
name_label = f"{u['name']} ({arc_count} file{'s' if arc_count != 1 else ''})"
|
|
else:
|
|
name_label = u["name"]
|
|
print(f" {prefix}: {u['emulator']} needs {name_label}")
|
|
checks = u.get("checks", [])
|
|
if checks:
|
|
if verbose:
|
|
expected = u.get("expected", {})
|
|
if expected:
|
|
vals = ",".join(f"{k}={v}" for k, v in sorted(expected.items()))
|
|
ref_part = f" [{u['source_ref']}]" if u.get("source_ref") else ""
|
|
print(f" validates {vals}{ref_part}")
|
|
else:
|
|
checks_label = "+".join(checks)
|
|
ref_part = f" [{u['source_ref']}]" if u.get("source_ref") else ""
|
|
print(f" validates {checks_label}{ref_part}")
|
|
else:
|
|
print(f" [{'+'.join(checks)}]")
|
|
|
|
|
|
def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
|
"""Print cross-reference section for undeclared files used by cores."""
|
|
undeclared = result.get("undeclared_files", [])
|
|
if not undeclared:
|
|
return
|
|
|
|
bios_files = [u for u in undeclared if u.get("category", "bios") == "bios"]
|
|
game_data = [u for u in undeclared if u.get("category", "bios") == "game_data"]
|
|
|
|
req_not_in_repo = [
|
|
u
|
|
for u in bios_files
|
|
if u["required"] and not u["in_repo"] and not u.get("hle_fallback")
|
|
]
|
|
req_hle_not_in_repo = [
|
|
u
|
|
for u in bios_files
|
|
if u["required"] and not u["in_repo"] and u.get("hle_fallback")
|
|
]
|
|
req_in_repo = [u for u in bios_files if u["required"] and u["in_repo"]]
|
|
opt_in_repo = [u for u in bios_files if not u["required"] and u["in_repo"]]
|
|
opt_not_in_repo = [u for u in bios_files if not u["required"] and not u["in_repo"]]
|
|
|
|
core_in_pack = len(req_in_repo) + len(opt_in_repo)
|
|
core_missing_req = len(req_not_in_repo) + len(req_hle_not_in_repo)
|
|
core_missing_opt = len(opt_not_in_repo)
|
|
|
|
print(
|
|
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, {core_missing_opt} optional missing"
|
|
)
|
|
|
|
for u in req_not_in_repo:
|
|
_print_undeclared_entry(u, "MISSING (required)", verbose)
|
|
for u in req_hle_not_in_repo:
|
|
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose)
|
|
|
|
if game_data:
|
|
gd_missing = [u for u in game_data if not u["in_repo"]]
|
|
gd_present = [u for u in game_data if u["in_repo"]]
|
|
if gd_missing or gd_present:
|
|
print(f" Game data: {len(gd_present)} in pack, {len(gd_missing)} missing")
|
|
|
|
|
|
def print_platform_result(
|
|
result: dict, group: list[str], verbose: bool = False
|
|
) -> None:
|
|
mode = result["verification_mode"]
|
|
total = result["total_files"]
|
|
c = result["severity_counts"]
|
|
label = " / ".join(group)
|
|
ok_count = c[Severity.OK]
|
|
problems = total - ok_count
|
|
|
|
# Summary line
|
|
if not reads_file_contents(mode):
|
|
if problems:
|
|
missing = c.get(Severity.WARNING, 0) + c.get(Severity.CRITICAL, 0)
|
|
optional_missing = c.get(Severity.INFO, 0)
|
|
parts = [f"{ok_count}/{total} present"]
|
|
if missing:
|
|
parts.append(f"{missing} missing")
|
|
if optional_missing:
|
|
parts.append(f"{optional_missing} optional missing")
|
|
else:
|
|
parts = [f"{ok_count}/{total} present"]
|
|
else:
|
|
sc = result.get("status_counts", {})
|
|
untested = sc.get(Status.UNTESTED, 0)
|
|
missing = sc.get(Status.MISSING, 0)
|
|
parts = [f"{ok_count}/{total} OK"]
|
|
if untested:
|
|
parts.append(f"{untested} untested")
|
|
if missing:
|
|
parts.append(f"{missing} missing")
|
|
print(f"{label}: {', '.join(parts)} [{mode}]")
|
|
|
|
seen_details: set[str] = set()
|
|
_print_detail_entries(result["details"], seen_details, verbose)
|
|
_print_undeclared_section(result, verbose)
|
|
|
|
exclusions = result.get("exclusion_notes", [])
|
|
if exclusions:
|
|
print(f" No external files ({len(exclusions)}):")
|
|
for ex in exclusions:
|
|
print(f" {ex['emulator']} -{ex['detail']} [{ex['reason']}]")
|
|
|
|
gt_cov = result.get("ground_truth_coverage")
|
|
if gt_cov and gt_cov["total"] > 0:
|
|
pct = gt_cov["with_validation"] * 100 // gt_cov["total"]
|
|
print(
|
|
f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)"
|
|
)
|
|
if gt_cov["platform_only"]:
|
|
print(f" {gt_cov['platform_only']} platform-only (no emulator profile)")
|
|
|
|
|
|
# Emulator/system mode verification
|
|
|
|
|
|
def _effective_validation_label(details: list[dict], validation_index: dict) -> str:
|
|
"""Determine the bracket label for the report.
|
|
|
|
Returns the union of all check types used, e.g. [crc32+existence+size].
|
|
"""
|
|
all_checks: set[str] = set()
|
|
has_files = False
|
|
for d in details:
|
|
fname = d.get("name", "")
|
|
if d.get("note"):
|
|
continue # skip informational entries (empty profiles)
|
|
has_files = True
|
|
entry = validation_index.get(fname)
|
|
if entry:
|
|
all_checks.update(entry["checks"])
|
|
else:
|
|
all_checks.add("existence")
|
|
if not has_files:
|
|
return "existence"
|
|
return "+".join(sorted(all_checks))
|
|
|
|
|
|
def verify_emulator(
|
|
profile_names: list[str],
|
|
emulators_dir: str,
|
|
db: dict,
|
|
standalone: bool = False,
|
|
regions: list[str] | None = None,
|
|
) -> dict:
|
|
"""Verify files for specific emulator profiles.
|
|
|
|
A region priority list narrows the report the same way a pack built with the
|
|
same list would be narrowed. One group per profile, as in generate_pack.
|
|
"""
|
|
load_emulator_profiles(emulators_dir)
|
|
zip_contents = build_zip_contents_index(db)
|
|
|
|
# Also load aliases for redirect messages
|
|
all_profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
|
|
|
|
# Resolve profile names, reject alias/launcher
|
|
selected: list[tuple[str, dict]] = []
|
|
for name in profile_names:
|
|
if name not in all_profiles:
|
|
available = sorted(
|
|
k
|
|
for k, v in all_profiles.items()
|
|
if v.get("type") not in ("alias", "test")
|
|
)
|
|
print(f"Error: emulator '{name}' not found", file=sys.stderr)
|
|
print(f"Available: {', '.join(available[:10])}...", file=sys.stderr)
|
|
sys.exit(1)
|
|
p = all_profiles[name]
|
|
if p.get("type") == "alias":
|
|
alias_of = p.get("alias_of", "?")
|
|
print(
|
|
f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
if p.get("type") == "launcher":
|
|
print(
|
|
f"Error: {name} is a launcher -use the emulator it launches",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
# Check standalone capability
|
|
ptype = p.get("type", "libretro")
|
|
if standalone and "standalone" not in ptype:
|
|
print(
|
|
f"Error: {name} ({ptype}) does not support --standalone",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
selected.append((name, p))
|
|
|
|
# Build validation index from selected profiles only
|
|
selected_profiles = {n: p for n, p in selected}
|
|
validation_index = _build_validation_index(selected_profiles)
|
|
data_registry = load_data_dir_registry(
|
|
os.path.join(os.path.dirname(__file__), "..", "platforms")
|
|
)
|
|
|
|
details = []
|
|
file_status: dict[str, str] = {}
|
|
file_severity: dict[str, str] = {}
|
|
dest_to_name: dict[str, str] = {}
|
|
data_dir_notices: list[str] = []
|
|
|
|
region_drops: set[str] = set()
|
|
if regions:
|
|
import region as region_mod
|
|
|
|
region_index = region_mod.build_region_index(dict(selected))
|
|
region_groups: dict[str, list[tuple[str, str]]] = {}
|
|
for emu_name, profile in selected:
|
|
members = region_groups.setdefault(emu_name, [])
|
|
for fe in filter_files_by_mode(profile.get("files", []), standalone):
|
|
nm = fe.get("name", "")
|
|
key = fe.get("path") or nm
|
|
if key:
|
|
members.append((key, nm))
|
|
region_drops = region_mod.resolve_region_drops(
|
|
region_groups, region_index, regions
|
|
)
|
|
|
|
for emu_name, profile in selected:
|
|
files = filter_files_by_mode(profile.get("files", []), standalone)
|
|
if region_drops:
|
|
files = [
|
|
fe
|
|
for fe in files
|
|
if (fe.get("path") or fe.get("name", "")) not in region_drops
|
|
]
|
|
|
|
# Check data directories (only notice if not cached)
|
|
for dd in profile.get("data_directories", []):
|
|
ref = dd.get("ref", "")
|
|
if not ref:
|
|
continue
|
|
if data_registry and ref in data_registry:
|
|
cache_path = data_registry[ref].get("local_cache", "")
|
|
if cache_path and os.path.isdir(cache_path):
|
|
continue # cached, no notice needed
|
|
data_dir_notices.append(ref)
|
|
|
|
if not files:
|
|
details.append(
|
|
{
|
|
"name": f"({emu_name})",
|
|
"status": Status.OK,
|
|
"required": False,
|
|
"system": "",
|
|
"note": f"No files needed for {profile.get('emulator', emu_name)}",
|
|
"ground_truth": [],
|
|
}
|
|
)
|
|
continue
|
|
|
|
# Verify archives as units (e.g., neogeo.zip, aes.zip)
|
|
seen_archives: set[str] = set()
|
|
for file_entry in files:
|
|
archive = file_entry.get("archive")
|
|
if archive and archive not in seen_archives:
|
|
seen_archives.add(archive)
|
|
# Prefer the profile's own entry for the archive (carries
|
|
# hashes for exact resolution when several dumps share a name)
|
|
archive_entry = next(
|
|
(
|
|
f
|
|
for f in files
|
|
if f.get("name") == archive and not f.get("archive")
|
|
),
|
|
{"name": archive},
|
|
)
|
|
local_path, _ = resolve_local_file(
|
|
archive_entry,
|
|
db,
|
|
zip_contents,
|
|
data_dir_registry=data_registry,
|
|
)
|
|
required = any(
|
|
f.get("archive") == archive and f.get("required", True)
|
|
for f in files
|
|
)
|
|
if local_path:
|
|
result = {
|
|
"name": archive,
|
|
"status": Status.OK,
|
|
"required": required,
|
|
"path": local_path,
|
|
}
|
|
else:
|
|
result = {
|
|
"name": archive,
|
|
"status": Status.MISSING,
|
|
"required": required,
|
|
}
|
|
result["system"] = file_entry.get("system", "")
|
|
result["hle_fallback"] = False
|
|
result["ground_truth"] = build_ground_truth(archive, validation_index)
|
|
details.append(result)
|
|
dest = archive
|
|
dest_to_name[dest] = archive
|
|
cur = result["status"]
|
|
prev = file_status.get(dest)
|
|
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(
|
|
prev, 0
|
|
):
|
|
file_status[dest] = cur
|
|
sev = compute_severity(cur, required, "existence", False)
|
|
prev_sev = file_severity.get(dest)
|
|
if prev_sev is None or _SEVERITY_ORDER.get(
|
|
sev, 0
|
|
) > _SEVERITY_ORDER.get(prev_sev, 0):
|
|
file_severity[dest] = sev
|
|
|
|
for file_entry in files:
|
|
# Skip archived files (verified as archive units above)
|
|
if file_entry.get("archive"):
|
|
continue
|
|
|
|
dest_hint = file_entry.get("path", "")
|
|
local_path, resolve_status = resolve_local_file(
|
|
file_entry,
|
|
db,
|
|
zip_contents,
|
|
dest_hint=dest_hint,
|
|
data_dir_registry=data_registry,
|
|
)
|
|
name = file_entry.get("name", "")
|
|
required = file_entry.get("required", True)
|
|
hle = file_entry.get("hle_fallback", False)
|
|
|
|
if not local_path:
|
|
result = {"name": name, "status": Status.MISSING, "required": required}
|
|
# An entry the profile documents as unsourceable is absent by
|
|
# design: a per-user key, a slot the user fills, a dump nobody
|
|
# has made. Counting it beside a file somebody could still
|
|
# find invites the wrong repair -- dropping the flag, or
|
|
# chasing a vendor's whole install tree.
|
|
unsourceable = file_entry.get("unsourceable")
|
|
if unsourceable:
|
|
result["unsourceable"] = (
|
|
unsourceable if isinstance(unsourceable, str) else ""
|
|
)
|
|
else:
|
|
# Apply emulator validation
|
|
check = check_file_validation(local_path, name, validation_index)
|
|
if check:
|
|
reason, _emus = check
|
|
better = _find_best_variant(
|
|
file_entry, db, local_path, validation_index,
|
|
)
|
|
if better:
|
|
result = {
|
|
"name": name,
|
|
"status": Status.OK,
|
|
"required": required,
|
|
"path": better,
|
|
}
|
|
else:
|
|
result = {
|
|
"name": name,
|
|
"status": Status.UNTESTED,
|
|
"required": required,
|
|
"path": local_path,
|
|
"reason": reason,
|
|
}
|
|
elif resolve_status == "hash_mismatch":
|
|
# Nothing in validation: caught it, but the name matched
|
|
# while the bytes contradict the hash the profile
|
|
# declares. That is a different file wearing the right
|
|
# name -- config.ini and ROM collide across systems -- so
|
|
# calling it covered reports the collection as holding
|
|
# something it does not. Validation runs first because it
|
|
# names the specific field that disagrees.
|
|
result = {
|
|
"name": name,
|
|
"status": Status.UNTESTED,
|
|
"required": required,
|
|
"path": local_path,
|
|
"reason": "declared hash contradicted by the local file",
|
|
}
|
|
else:
|
|
result = {
|
|
"name": name,
|
|
"status": Status.OK,
|
|
"required": required,
|
|
"path": local_path,
|
|
}
|
|
|
|
result["system"] = file_entry.get("system", "")
|
|
result["hle_fallback"] = hle
|
|
result["ground_truth"] = build_ground_truth(name, validation_index)
|
|
details.append(result)
|
|
|
|
# Aggregate by destination (path if available, else name)
|
|
dest = file_entry.get("path", "") or name
|
|
dest_to_name[dest] = name
|
|
cur = result["status"]
|
|
prev = file_status.get(dest)
|
|
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
|
|
file_status[dest] = cur
|
|
sev = compute_severity(cur, required, "existence", hle)
|
|
prev_sev = file_severity.get(dest)
|
|
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(
|
|
prev_sev, 0
|
|
):
|
|
file_severity[dest] = sev
|
|
|
|
counts = {
|
|
Severity.OK: 0,
|
|
Severity.INFO: 0,
|
|
Severity.WARNING: 0,
|
|
Severity.CRITICAL: 0,
|
|
}
|
|
for s in file_severity.values():
|
|
counts[s] = counts.get(s, 0) + 1
|
|
status_counts: dict[str, int] = {}
|
|
for s in file_status.values():
|
|
status_counts[s] = status_counts.get(s, 0) + 1
|
|
|
|
label = _effective_validation_label(details, validation_index)
|
|
|
|
gt_filenames = set(validation_index)
|
|
total = len(file_status)
|
|
with_validation = sum(
|
|
1 for dest in file_status if dest_to_name.get(dest, "") in gt_filenames
|
|
)
|
|
|
|
return {
|
|
"emulators": [n for n, _ in selected],
|
|
"verification_mode": label,
|
|
"total_files": total,
|
|
"severity_counts": counts,
|
|
"status_counts": status_counts,
|
|
"details": details,
|
|
"data_dir_notices": sorted(set(data_dir_notices)),
|
|
"ground_truth_coverage": {
|
|
"with_validation": with_validation,
|
|
"platform_only": total - with_validation,
|
|
"total": total,
|
|
},
|
|
}
|
|
|
|
|
|
def verify_system(
|
|
system_ids: list[str],
|
|
emulators_dir: str,
|
|
db: dict,
|
|
standalone: bool = False,
|
|
regions: list[str] | None = None,
|
|
) -> dict:
|
|
"""Verify files for all emulators supporting given system IDs."""
|
|
profiles = load_emulator_profiles(emulators_dir)
|
|
matching = []
|
|
for name, profile in sorted(profiles.items()):
|
|
if profile.get("type") in ("launcher", "alias", "test"):
|
|
continue
|
|
emu_systems = set(profile.get("systems", []))
|
|
if emu_systems & set(system_ids):
|
|
ptype = profile.get("type", "libretro")
|
|
if standalone and "standalone" not in ptype:
|
|
continue # skip non-standalone in standalone mode
|
|
matching.append(name)
|
|
|
|
if not matching:
|
|
all_systems: set[str] = set()
|
|
for p in profiles.values():
|
|
all_systems.update(p.get("systems", []))
|
|
if standalone:
|
|
print(
|
|
f"No standalone emulators found for system(s): {', '.join(system_ids)}",
|
|
file=sys.stderr,
|
|
)
|
|
else:
|
|
print(
|
|
f"No emulators found for system(s): {', '.join(system_ids)}",
|
|
file=sys.stderr,
|
|
)
|
|
print(
|
|
f"Available systems: {', '.join(sorted(all_systems)[:20])}...",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(1)
|
|
|
|
return verify_emulator(matching, emulators_dir, db, standalone, regions=regions)
|
|
|
|
|
|
def print_emulator_result(result: dict, verbose: bool = False) -> None:
|
|
"""Print verification result for emulator/system mode."""
|
|
label = " + ".join(result["emulators"])
|
|
mode = result["verification_mode"]
|
|
total = result["total_files"]
|
|
c = result["severity_counts"]
|
|
ok_count = c[Severity.OK]
|
|
|
|
sc = result.get("status_counts", {})
|
|
untested = sc.get(Status.UNTESTED, 0)
|
|
missing = sc.get(Status.MISSING, 0)
|
|
unsourceable_names = {
|
|
d["name"]
|
|
for d in result["details"]
|
|
if d["status"] == Status.MISSING and "unsourceable" in d
|
|
}
|
|
unsourceable = len(unsourceable_names)
|
|
parts = [f"{ok_count}/{total} OK"]
|
|
if untested:
|
|
parts.append(f"{untested} untested")
|
|
if missing - unsourceable > 0:
|
|
parts.append(f"{missing - unsourceable} missing")
|
|
if unsourceable:
|
|
parts.append(f"{unsourceable} unsourceable")
|
|
print(f"{label}: {', '.join(parts)} [{mode}]")
|
|
|
|
seen = set()
|
|
for d in result["details"]:
|
|
if d["status"] == Status.UNTESTED:
|
|
if d["name"] in seen:
|
|
continue
|
|
seen.add(d["name"])
|
|
req = "required" if d.get("required", True) else "optional"
|
|
hle = ", HLE available" if d.get("hle_fallback") else ""
|
|
reason = d.get("reason", "")
|
|
print(f" UNTESTED ({req}{hle}): {d['name']} -{reason}")
|
|
gt = d.get("ground_truth", [])
|
|
if gt:
|
|
if verbose:
|
|
for line in _format_ground_truth_verbose(gt):
|
|
print(f" {line}")
|
|
else:
|
|
print(f" {_format_ground_truth_aggregate(gt)}")
|
|
for d in result["details"]:
|
|
if d["status"] == Status.MISSING and "unsourceable" not in d:
|
|
if d["name"] in seen:
|
|
continue
|
|
seen.add(d["name"])
|
|
req = "required" if d.get("required", True) else "optional"
|
|
hle = ", HLE available" if d.get("hle_fallback") else ""
|
|
print(f" MISSING ({req}{hle}): {d['name']}")
|
|
gt = d.get("ground_truth", [])
|
|
if gt:
|
|
if verbose:
|
|
for line in _format_ground_truth_verbose(gt):
|
|
print(f" {line}")
|
|
else:
|
|
print(f" {_format_ground_truth_aggregate(gt)}")
|
|
for d in result["details"]:
|
|
if d["status"] == Status.MISSING and "unsourceable" in d:
|
|
if d["name"] in seen:
|
|
continue
|
|
seen.add(d["name"])
|
|
why = d.get("unsourceable") or "documented as unobtainable"
|
|
print(f" UNSOURCEABLE: {d['name']} -{why}")
|
|
|
|
for d in result["details"]:
|
|
if d.get("note"):
|
|
print(f" {d['note']}")
|
|
|
|
if verbose:
|
|
for d in result["details"]:
|
|
if d["status"] == Status.OK:
|
|
if d["name"] in seen:
|
|
continue
|
|
seen.add(d["name"])
|
|
gt = d.get("ground_truth", [])
|
|
if gt:
|
|
req = "required" if d.get("required", True) else "optional"
|
|
print(f" OK ({req}): {d['name']}")
|
|
for line in _format_ground_truth_verbose(gt):
|
|
print(f" {line}")
|
|
|
|
for ref in result.get("data_dir_notices", []):
|
|
print(
|
|
f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)"
|
|
)
|
|
|
|
# Ground truth coverage footer
|
|
gt_cov = result.get("ground_truth_coverage")
|
|
if gt_cov and gt_cov["total"] > 0:
|
|
pct = gt_cov["with_validation"] * 100 // gt_cov["total"]
|
|
print(
|
|
f" Ground truth: {gt_cov['with_validation']}/{gt_cov['total']} files have emulator validation ({pct}%)"
|
|
)
|
|
if gt_cov["platform_only"]:
|
|
print(f" {gt_cov['platform_only']} without declared validation")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Platform-native BIOS verification")
|
|
parser.add_argument("--platform", "-p", help="Platform name")
|
|
parser.add_argument(
|
|
"--all", action="store_true", help="Verify all active platforms"
|
|
)
|
|
parser.add_argument(
|
|
"--emulator", "-e", help="Emulator profile name(s), comma-separated"
|
|
)
|
|
parser.add_argument("--system", "-s", help="System ID(s), comma-separated")
|
|
parser.add_argument("--standalone", action="store_true", help="Use standalone mode")
|
|
parser.add_argument(
|
|
"--list-emulators", action="store_true", help="List available emulators"
|
|
)
|
|
parser.add_argument(
|
|
"--list-systems", action="store_true", help="List available systems"
|
|
)
|
|
parser.add_argument("--include-archived", action="store_true")
|
|
parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)")
|
|
parser.add_argument(
|
|
"--region", help="Region priority list, best first (e.g. us,eu,jp)"
|
|
)
|
|
parser.add_argument(
|
|
"--list-targets",
|
|
action="store_true",
|
|
help="List available targets for the platform",
|
|
)
|
|
parser.add_argument("--db", default=DEFAULT_DB)
|
|
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
|
|
parser.add_argument("--emulators-dir", default=DEFAULT_EMULATORS_DIR)
|
|
parser.add_argument(
|
|
"--verbose",
|
|
"-v",
|
|
action="store_true",
|
|
help="Show emulator ground truth details",
|
|
)
|
|
parser.add_argument("--json", action="store_true", help="JSON output")
|
|
args = parser.parse_args()
|
|
|
|
requested_regions: list[str] = []
|
|
if getattr(args, "region", None):
|
|
import region as region_mod
|
|
|
|
try:
|
|
requested_regions = region_mod.parse_requested(args.region)
|
|
except ValueError as exc:
|
|
parser.error(str(exc))
|
|
|
|
if args.list_emulators:
|
|
list_emulator_profiles(args.emulators_dir)
|
|
return
|
|
if args.list_systems:
|
|
list_system_ids(args.emulators_dir)
|
|
return
|
|
|
|
if args.list_targets:
|
|
if not args.platform:
|
|
parser.error("--list-targets requires --platform")
|
|
from common import list_available_targets
|
|
|
|
targets = list_available_targets(args.platform, args.platforms_dir)
|
|
if not targets:
|
|
print(f"No targets configured for platform '{args.platform}'")
|
|
return
|
|
for t in targets:
|
|
aliases = f" (aliases: {', '.join(t['aliases'])})" if t["aliases"] else ""
|
|
print(
|
|
f" {t['name']:30s} {t['architecture']:10s} {t['core_count']:>4d} cores{aliases}"
|
|
)
|
|
return
|
|
|
|
# Mutual exclusion
|
|
modes = sum(1 for x in (args.platform, args.all, args.emulator, args.system) if x)
|
|
if modes == 0:
|
|
parser.error("Specify --platform, --all, --emulator, or --system")
|
|
if modes > 1:
|
|
parser.error(
|
|
"--platform, --all, --emulator, and --system are mutually exclusive"
|
|
)
|
|
if args.standalone and not (args.emulator or args.system):
|
|
parser.error("--standalone requires --emulator or --system")
|
|
if args.target and not (args.platform or args.all):
|
|
parser.error("--target requires --platform or --all")
|
|
if args.target and (args.emulator or args.system):
|
|
parser.error("--target is incompatible with --emulator and --system")
|
|
|
|
with open(args.db) as f:
|
|
db = json.load(f)
|
|
|
|
# Emulator mode
|
|
if args.emulator:
|
|
names = [n.strip() for n in args.emulator.split(",") if n.strip()]
|
|
result = verify_emulator(
|
|
names, args.emulators_dir, db, args.standalone,
|
|
regions=requested_regions,
|
|
)
|
|
if args.json:
|
|
result["details"] = [
|
|
d for d in result["details"] if d["status"] != Status.OK
|
|
]
|
|
print(json.dumps(result, indent=2))
|
|
else:
|
|
print_emulator_result(result, verbose=args.verbose)
|
|
return
|
|
|
|
# System mode
|
|
if args.system:
|
|
system_ids = [s.strip() for s in args.system.split(",") if s.strip()]
|
|
result = verify_system(
|
|
system_ids, args.emulators_dir, db, args.standalone,
|
|
regions=requested_regions,
|
|
)
|
|
if args.json:
|
|
result["details"] = [
|
|
d for d in result["details"] if d["status"] != Status.OK
|
|
]
|
|
print(json.dumps(result, indent=2))
|
|
else:
|
|
print_emulator_result(result, verbose=args.verbose)
|
|
return
|
|
|
|
# Platform mode (existing)
|
|
if args.all:
|
|
from list_platforms import list_platforms as _list_platforms
|
|
|
|
platforms = _list_platforms(include_archived=args.include_archived)
|
|
elif args.platform:
|
|
platforms = [args.platform]
|
|
else:
|
|
parser.error("Specify --platform or --all")
|
|
return
|
|
|
|
# Load emulator profiles once for cross-reference (not per-platform)
|
|
emu_profiles = load_emulator_profiles(args.emulators_dir)
|
|
data_registry = load_data_dir_registry(args.platforms_dir)
|
|
|
|
target_cores_cache: dict[str, set[str] | None] = {}
|
|
if args.target:
|
|
try:
|
|
target_cores_cache, platforms = build_target_cores_cache(
|
|
platforms,
|
|
args.target,
|
|
args.platforms_dir,
|
|
is_all=args.all,
|
|
)
|
|
except (FileNotFoundError, ValueError) as e:
|
|
print(f"ERROR: {e}", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
# Group identical platforms (same function as generate_pack)
|
|
groups = group_identical_platforms(
|
|
platforms, args.platforms_dir, target_cores_cache if args.target else None
|
|
)
|
|
from cross_reference import _build_supplemental_index
|
|
|
|
suppl_names = _build_supplemental_index()
|
|
|
|
all_results = {}
|
|
group_results: list[tuple[dict, list[str]]] = []
|
|
for group_platforms, representative in groups:
|
|
config = load_platform_config(representative, args.platforms_dir)
|
|
tc = target_cores_cache.get(representative) if args.target else None
|
|
result = verify_platform(
|
|
config,
|
|
db,
|
|
args.emulators_dir,
|
|
emu_profiles,
|
|
target_cores=tc,
|
|
data_dir_registry=data_registry,
|
|
supplemental_names=suppl_names,
|
|
regions=requested_regions,
|
|
)
|
|
names = [
|
|
load_platform_config(p, args.platforms_dir).get("platform", p)
|
|
for p in group_platforms
|
|
]
|
|
group_results.append((result, names))
|
|
for p in group_platforms:
|
|
all_results[p] = result
|
|
|
|
if not args.json:
|
|
for result, group in group_results:
|
|
print_platform_result(result, group, verbose=args.verbose)
|
|
print()
|
|
|
|
if args.json:
|
|
for r in all_results.values():
|
|
r["details"] = [d for d in r["details"] if d["status"] != Status.OK]
|
|
print(json.dumps(all_results, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|