feat: add emulator/system pack generation, validation checks, path resolution

add --emulator, --system, --standalone, --list-emulators, --list-systems
to verify.py and generate_pack.py. packs are RTU with data directories,
regional BIOS variants, and archive support.

validation: field per file (size, crc32, md5, sha1) with conflict
detection. by_path_suffix index in database.json for regional variant
resolution via dest_hint. restructure GameCube IPL to regional subdirs.

66 E2E tests, full pipeline verified.
This commit is contained in:
Abdessamad Derraz
2026-03-22 14:02:20 +01:00
parent d2adde9846
commit 1d350f0578
21 changed files with 17218 additions and 150 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

File diff suppressed because it is too large Load Diff

View File

@@ -14,6 +14,15 @@ systems: [nintendo-gamecube, nintendo-wii]
# Standalone: User/GC/ and User/Wii/
# Libretro (RetroArch): system/dolphin-emu/GC/ and system/dolphin-emu/Wii/
pack_structure:
libretro: "dolphin-emu"
standalone: ""
data_directories:
- ref: dolphin-sys
destination: "Sys"
source_ref: "Source/Core/Common/CommonPaths.h:128-141"
files:
# -- GameCube IPL (Boot ROM) --
# Region-specific, placed in GC/<region>/IPL.bin

View File

@@ -1,7 +1,9 @@
emulator: gpSP
type: libretro
core_classification: community_fork
source: "https://github.com/libretro/gpsp"
profiled_date: "2026-03-18"
upstream: "https://github.com/BASLQC/gPSP"
profiled_date: "2026-03-21"
core_version: "Git"
display_name: "Nintendo - Game Boy Advance (gpSP)"
cores:
@@ -35,6 +37,9 @@ notes: |
precise SWI behavior or BIOS checksum verification. The built-in BIOS
works for the vast majority of the GBA library.
Upstream (Exophase v0.9) required the official BIOS with no fallback.
The built-in open-source BIOS is a libretro port addition.
files:
# -------------------------------------------------------
# Game Boy Advance - BIOS (optional, built-in fallback)
@@ -44,6 +49,7 @@ files:
required: false
hle_fallback: true
size: 16384 # 16 KB (0x4000)
validation: [size]
note: "Official GBA BIOS. Built-in open-source BIOS used as fallback. Real BIOS needed for full SWI accuracy and boot logo."
source_ref: "libretro/libretro.c:1111"

View File

@@ -1,9 +1,12 @@
emulator: Handy
type: libretro
core_classification: community_fork
source: "https://github.com/libretro/libretro-handy"
profiled_date: "2026-03-18"
profiled_date: "2026-03-21"
core_version: "0.95"
display_name: "Atari - Lynx (Handy)"
cores:
- handy
systems: [atari-lynx]
notes: |
@@ -36,8 +39,7 @@ files:
required: false
hle_fallback: true
size: 512
md5: "fcd403db69f54290b51035d82f835e7b"
sha1: "e4ed47fae31693e016b081c6bda48da5b70d7ccb"
crc32: "0d973c9d"
validation: [size, crc32]
source_ref: "rom.h:48-49 (ROM_SIZE=0x200, ROM_CRC32=0xD973C9D), rom.cpp:76-128 (CRom constructor), libretro.cpp:1231-1258 (bios path + CSystem init)"
notes: "Validated by CRC32 at load time. If invalid or missing, core uses HLE fallback (system.cpp HLE_BIOS_* functions). Games work without it but the real boot ROM provides accurate startup timing."

View File

@@ -1,10 +1,13 @@
emulator: Ishiiruka
type: libretro
core_classification: enhanced_fork
source: "https://github.com/libretro/ishiiruka"
profiled_date: "2026-03-18"
upstream: "https://github.com/Tinob/Ishiiruka"
profiled_date: "2026-03-21"
core_version: "Git"
display_name: "Nintendo - GameCube / Wii (Ishiiruka)"
fork_of: Dolphin
cores:
- ishiiruka
systems: [nintendo-gamecube, nintendo-wii]
# Ishiiruka is a performance-focused Dolphin fork with custom GPU backends.
@@ -19,6 +22,11 @@ systems: [nintendo-gamecube, nintendo-wii]
# Core options use "ishiiruka_" prefix (ishiiruka_dsp_hle, ishiiruka_efb_scale, etc.)
data_directories:
- ref: dolphin-sys
destination: "dolphin-emu/Sys"
source_ref: "Source/Core/Common/CommonPaths.h:115"
files:
# -- GameCube IPL (Boot ROM) --
# Region-specific, placed in GC/<region>/IPL.bin

View File

@@ -1,7 +1,9 @@
emulator: JollyCV
type: libretro
core_classification: official_port
source: "https://github.com/libretro/jollycv"
profiled_date: "2026-03-18"
upstream: "https://gitlab.com/jgemu/jollycv"
profiled_date: "2026-03-21"
core_version: "2.0.0"
display_name: "ColecoVision/CreatiVision/My Vision (JollyCV)"
cores:
@@ -47,11 +49,9 @@ files:
# --- ColecoVision BIOS (required for CV games) ---
- name: "coleco.rom"
system: colecovision
description: "ColecoVision BIOS ROM"
required: true
size: 8192
md5: "2c66f5911e5b42b8ebe113403548eee7"
sha1: "4aa1d9b48f39b68bb17b3d6997b74850c6089dc3"
validation: [size]
source_ref: "libretro/libretro.c:701, src/jcv_coleco.c:402-406"
notes: "Mapped at 0x0000-0x1FFF. Replaced by SGM lower RAM when Super Game Module is active. Must be exactly 8192 bytes."
@@ -61,6 +61,7 @@ files:
description: "VTech CreatiVision BIOS ROM"
required: true
size: 2048
validation: [size]
source_ref: "libretro/libretro.c:711, src/jcv_crvision.c:315-318"
notes: "Mapped at 0xF800-0xFFFF. Must be exactly 2048 bytes."

View File

@@ -40,6 +40,7 @@ files:
md5: "562d5ebf9e030a40d6fabfc2f33139fd"
sha1: "b2e1955d957a475de2411770452eff4ea19f4cee"
crc32: "8016a315"
validation: [size, crc32]
note: "Magnavox Odyssey2 BIOS (G7000 NTSC). Default BIOS, vpp=0."
source_ref: "libretro.c:182-186"
@@ -54,6 +55,7 @@ files:
md5: "f1071cdb0b6b10dde94d3bc8a6146387"
sha1: "a6120aed50831c9c0d95dbdf707820f601d9452e"
crc32: "a318e8d6"
validation: [size, crc32]
note: "Philips Videopac G7000 European BIOS. vpp=0, auto-sets PAL region."
source_ref: "libretro.c:192-197"
@@ -68,6 +70,7 @@ files:
md5: "c500ff71236068e0dc0d0603d265ae76"
sha1: "5130243429b40b01a14e1304d0394b8459a6fbae"
crc32: "e20a9f41"
validation: [size, crc32]
note: "Philips Videopac+ G7400 European BIOS. vpp=1, enables enhanced graphics."
source_ref: "libretro.c:187-191"
@@ -82,6 +85,7 @@ files:
md5: "279008e4a0db2dc5f1c048853b033828"
sha1: "54b8d2c1317628de51a85fc1c424423a986775e4"
crc32: "11647ca5"
validation: [size, crc32]
note: "Philips Videopac+ G7400 French BIOS (JoPac). vpp=1, enables enhanced graphics."
source_ref: "libretro.c:198-203"

View File

@@ -158,6 +158,7 @@ def resolve_local_file(
file_entry: dict,
db: dict,
zip_contents: dict | None = None,
dest_hint: str = "",
) -> tuple[str | None, str]:
"""Resolve a BIOS file to its local path using database.json.
@@ -165,6 +166,10 @@ def resolve_local_file(
and generate_pack.py. Does NOT handle storage tiers (external/user_provided)
or release assets - callers handle those.
dest_hint: optional destination path (e.g., "GC/USA/IPL.bin") used to
disambiguate when multiple files share the same name. Matched against
the by_path_suffix index built from the repo's directory structure.
Returns (local_path, status) where status is one of:
exact, zip_exact, hash_mismatch, not_found.
"""
@@ -179,6 +184,15 @@ def resolve_local_file(
files_db = db.get("files", {})
by_md5 = db.get("indexes", {}).get("by_md5", {})
by_name = db.get("indexes", {}).get("by_name", {})
by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
# 0. Path suffix exact match (for regional variants with same filename)
if dest_hint and by_path_suffix:
for match_sha1 in by_path_suffix.get(dest_hint, []):
if match_sha1 in files_db:
path = files_db[match_sha1]["path"]
if os.path.exists(path):
return path, "exact"
# 1. SHA1 exact match
if sha1 and sha1 in files_db:

View File

@@ -77,10 +77,23 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
"crc32": cached["crc32"],
}
sha1 = hashes["sha1"]
is_variant = "/.variants/" in rel_path or "\\.variants\\" in rel_path
if sha1 in files:
if sha1 not in aliases:
aliases[sha1] = []
aliases[sha1].append({"name": _canonical_name(filepath), "path": rel_path})
existing_is_variant = "/.variants/" in files[sha1]["path"]
if existing_is_variant and not is_variant:
if sha1 not in aliases:
aliases[sha1] = []
aliases[sha1].append({"name": files[sha1]["name"], "path": files[sha1]["path"]})
files[sha1] = {
"path": rel_path,
"name": _canonical_name(filepath),
"size": size,
**hashes,
}
else:
if sha1 not in aliases:
aliases[sha1] = []
aliases[sha1].append({"name": _canonical_name(filepath), "path": rel_path})
else:
entry = {
"path": rel_path,
@@ -94,10 +107,24 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
hashes = compute_hashes(filepath)
sha1 = hashes["sha1"]
is_variant = "/.variants/" in rel_path or "\\.variants\\" in rel_path
if sha1 in files:
if sha1 not in aliases:
aliases[sha1] = []
aliases[sha1].append({"name": _canonical_name(filepath), "path": rel_path})
existing_is_variant = "/.variants/" in files[sha1]["path"]
if existing_is_variant and not is_variant:
# Non-variant file should be primary over .variants/ file
if sha1 not in aliases:
aliases[sha1] = []
aliases[sha1].append({"name": files[sha1]["name"], "path": files[sha1]["path"]})
files[sha1] = {
"path": rel_path,
"name": _canonical_name(filepath),
"size": size,
**hashes,
}
else:
if sha1 not in aliases:
aliases[sha1] = []
aliases[sha1].append({"name": _canonical_name(filepath), "path": rel_path})
else:
entry = {
"path": rel_path,
@@ -111,11 +138,25 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
return files, aliases, new_cache
def _path_suffix(rel_path: str) -> str:
"""Extract the path suffix after bios/Manufacturer/Console/.
bios/Nintendo/GameCube/GC/USA/IPL.bin → GC/USA/IPL.bin
bios/Sony/PlayStation/scph5501.bin → scph5501.bin
"""
parts = rel_path.replace("\\", "/").split("/")
# Skip: bios / Manufacturer / Console (3 segments)
if len(parts) > 3 and parts[0] == "bios":
return "/".join(parts[3:])
return parts[-1]
def build_indexes(files: dict, aliases: dict) -> dict:
"""Build secondary indexes for fast lookup."""
by_md5 = {}
by_name = {}
by_crc32 = {}
by_path_suffix = {}
for sha1, entry in files.items():
by_md5[entry["md5"]] = sha1
@@ -127,6 +168,14 @@ def build_indexes(files: dict, aliases: dict) -> dict:
by_crc32[entry["crc32"]] = sha1
# Path suffix index for regional variant resolution
suffix = _path_suffix(entry["path"])
if suffix != name:
# Only index when suffix adds info beyond the filename
if suffix not in by_path_suffix:
by_path_suffix[suffix] = []
by_path_suffix[suffix].append(sha1)
# Add alias names to by_name index (aliases have different filenames for same SHA1)
for sha1, alias_list in aliases.items():
for alias in alias_list:
@@ -135,11 +184,19 @@ def build_indexes(files: dict, aliases: dict) -> dict:
by_name[name] = []
if sha1 not in by_name[name]:
by_name[name].append(sha1)
# Also index alias paths in by_path_suffix
suffix = _path_suffix(alias["path"])
if suffix != name:
if suffix not in by_path_suffix:
by_path_suffix[suffix] = []
if sha1 not in by_path_suffix[suffix]:
by_path_suffix[suffix].append(sha1)
return {
"by_md5": by_md5,
"by_name": by_name,
"by_crc32": by_crc32,
"by_path_suffix": by_path_suffix,
}

View File

@@ -100,7 +100,8 @@ def _sanitize_path(raw: str) -> str:
def resolve_file(file_entry: dict, db: dict, bios_dir: str,
zip_contents: dict | None = None) -> tuple[str | None, str]:
zip_contents: dict | None = None,
dest_hint: str = "") -> tuple[str | None, str]:
"""Resolve a BIOS file with storage tiers and release asset fallback.
Wraps common.resolve_local_file() with pack-specific logic for
@@ -112,7 +113,8 @@ def resolve_file(file_entry: dict, db: dict, bios_dir: str,
if storage == "external":
return None, "external"
path, status = resolve_local_file(file_entry, db, zip_contents)
path, status = resolve_local_file(file_entry, db, zip_contents,
dest_hint=dest_hint)
if path:
return path, status
@@ -466,6 +468,334 @@ def _extract_zip_to_archive(source_zip: str, dest_prefix: str, target_zf: zipfil
target_zf.writestr(target_path, data)
# ---------------------------------------------------------------------------
# Emulator/system mode pack generation
# ---------------------------------------------------------------------------
def _filter_files_by_mode(files: list[dict], standalone: bool) -> list[dict]:
"""Filter file entries by libretro/standalone mode."""
result = []
for f in files:
fmode = f.get("mode", "")
if standalone and fmode == "libretro":
continue
if not standalone and fmode == "standalone":
continue
result.append(f)
return result
def _resolve_destination(file_entry: dict, pack_structure: dict | None,
standalone: bool) -> str:
"""Resolve the ZIP destination path for a file entry."""
# 1. standalone_path override
if standalone and file_entry.get("standalone_path"):
rel = file_entry["standalone_path"]
# 2. path field
elif file_entry.get("path"):
rel = file_entry["path"]
# 3. name fallback
else:
rel = file_entry.get("name", "")
rel = _sanitize_path(rel)
# Prepend pack_structure prefix
if pack_structure:
mode_key = "standalone" if standalone else "libretro"
prefix = pack_structure.get(mode_key, "")
if prefix:
rel = f"{prefix}/{rel}"
return rel
def generate_emulator_pack(
profile_names: list[str],
emulators_dir: str,
db: dict,
bios_dir: str,
output_dir: str,
standalone: bool = False,
zip_contents: dict | None = None,
) -> str | None:
"""Generate a ZIP pack for specific emulator profiles."""
all_profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
if zip_contents is None:
zip_contents = build_zip_contents_index(db)
# Resolve and validate profile names
selected: list[tuple[str, dict]] = []
for name in profile_names:
if name not in all_profiles:
available = sorted(k for k, v in all_profiles.items()
if v.get("type") not in ("alias", "test"))
print(f"Error: emulator '{name}' not found", file=sys.stderr)
print(f"Available: {', '.join(available[:10])}...", file=sys.stderr)
return None
p = all_profiles[name]
if p.get("type") == "alias":
alias_of = p.get("alias_of", "?")
print(f"Error: {name} is an alias of {alias_of} — use --emulator {alias_of}",
file=sys.stderr)
return None
if p.get("type") == "launcher":
print(f"Error: {name} is a launcher — use the emulator it launches",
file=sys.stderr)
return None
ptype = p.get("type", "libretro")
if standalone and "standalone" not in ptype:
print(f"Error: {name} ({ptype}) does not support --standalone",
file=sys.stderr)
return None
selected.append((name, p))
# ZIP naming
display_names = [p.get("emulator", n).replace(" ", "") for n, p in selected]
zip_name = "_".join(display_names) + "_BIOS_Pack.zip"
zip_path = os.path.join(output_dir, zip_name)
os.makedirs(output_dir, exist_ok=True)
total_files = 0
missing_files = []
seen_destinations: set[str] = set()
seen_lower: set[str] = set()
seen_hashes: set[str] = set() # SHA1 dedup for same file, different path
data_dir_notices: list[str] = []
data_registry = load_data_dir_registry(
os.path.join(os.path.dirname(__file__), "..", "platforms")
)
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for emu_name, profile in sorted(selected):
pack_structure = profile.get("pack_structure")
files = _filter_files_by_mode(profile.get("files", []), standalone)
for dd in profile.get("data_directories", []):
ref_key = dd.get("ref", "")
if not ref_key or not data_registry or ref_key not in data_registry:
if ref_key:
data_dir_notices.append(ref_key)
continue
entry = data_registry[ref_key]
local_cache = entry.get("local_cache", "")
if not local_cache or not os.path.isdir(local_cache):
data_dir_notices.append(ref_key)
continue
dd_dest = dd.get("destination", "")
if pack_structure:
mode_key = "standalone" if standalone else "libretro"
prefix = pack_structure.get(mode_key, "")
if prefix:
dd_dest = f"{prefix}/{dd_dest}" if dd_dest else prefix
for root, _dirs, filenames in os.walk(local_cache):
for fname in filenames:
src = os.path.join(root, fname)
rel = os.path.relpath(src, local_cache)
full = f"{dd_dest}/{rel}" if dd_dest else rel
if full.lower() in seen_lower:
continue
seen_destinations.add(full)
seen_lower.add(full.lower())
zf.write(src, full)
total_files += 1
if not files:
print(f" No files needed for {profile.get('emulator', emu_name)}")
continue
# Collect archives as atomic units
archives: set[str] = set()
for fe in files:
archive = fe.get("archive")
if archive:
archives.add(archive)
# Pack archives as units
for archive_name in sorted(archives):
archive_dest = _sanitize_path(archive_name)
if pack_structure:
mode_key = "standalone" if standalone else "libretro"
prefix = pack_structure.get(mode_key, "")
if prefix:
archive_dest = f"{prefix}/{archive_dest}"
if archive_dest.lower() in seen_lower:
continue
archive_entry = {"name": archive_name}
local_path, status = resolve_file(archive_entry, db, bios_dir, zip_contents)
if local_path and status not in ("not_found",):
zf.write(local_path, archive_dest)
seen_destinations.add(archive_dest)
seen_lower.add(archive_dest.lower())
total_files += 1
else:
missing_files.append(archive_name)
# Pack individual files (skip archived ones)
for fe in files:
if fe.get("archive"):
continue
dest = _resolve_destination(fe, pack_structure, standalone)
if not dest:
continue
if dest.lower() in seen_lower:
continue
storage = fe.get("storage", "embedded")
if storage == "user_provided":
seen_destinations.add(dest)
seen_lower.add(dest.lower())
instr = fe.get("instructions", "Please provide this file manually.")
instr_name = f"INSTRUCTIONS_{fe['name']}.txt"
zf.writestr(instr_name, f"File needed: {fe['name']}\n\n{instr}\n")
total_files += 1
continue
dest_hint = fe.get("path", "")
local_path, status = resolve_file(fe, db, bios_dir, zip_contents,
dest_hint=dest_hint)
if status == "external":
file_ext = os.path.splitext(fe["name"])[1] or ""
with tempfile.NamedTemporaryFile(delete=False, suffix=file_ext) as tmp:
tmp_path = tmp.name
try:
if download_external(fe, tmp_path):
zf.write(tmp_path, dest)
seen_destinations.add(dest)
seen_lower.add(dest.lower())
total_files += 1
else:
missing_files.append(fe["name"])
finally:
if os.path.exists(tmp_path):
os.unlink(tmp_path)
continue
if status in ("not_found", "user_provided"):
missing_files.append(fe["name"])
continue
# SHA1 dedup: skip if same physical file AND same destination
# (but allow same file to be packed under different destinations,
# e.g., IPL.bin in GC/USA/ and GC/EUR/ from same source)
if local_path:
real = os.path.realpath(local_path)
dedup_key_hash = f"{real}:{dest}"
if dedup_key_hash in seen_hashes:
continue
seen_hashes.add(dedup_key_hash)
zf.write(local_path, dest)
seen_destinations.add(dest)
seen_lower.add(dest.lower())
total_files += 1
# Remove empty ZIP (no files packed and no missing = nothing to ship)
if total_files == 0 and not missing_files:
os.unlink(zip_path)
# Report
label = " + ".join(p.get("emulator", n) for n, p in selected)
missing_count = len(missing_files)
ok_count = total_files
parts = [f"{ok_count} files packed"]
if missing_count:
parts.append(f"{missing_count} missing")
print(f" {zip_path}: {', '.join(parts)}")
for name in missing_files:
print(f" MISSING: {name}")
for ref in sorted(set(data_dir_notices)):
print(f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)")
return zip_path if total_files > 0 or missing_files else None
def generate_system_pack(
system_ids: list[str],
emulators_dir: str,
db: dict,
bios_dir: str,
output_dir: str,
standalone: bool = False,
zip_contents: dict | None = None,
) -> str | None:
"""Generate a ZIP pack for all emulators supporting given system IDs."""
profiles = load_emulator_profiles(emulators_dir)
matching = []
for name, profile in sorted(profiles.items()):
if profile.get("type") in ("launcher", "alias", "test"):
continue
emu_systems = set(profile.get("systems", []))
if emu_systems & set(system_ids):
ptype = profile.get("type", "libretro")
if standalone and "standalone" not in ptype:
continue
matching.append(name)
if not matching:
all_systems: set[str] = set()
for p in profiles.values():
all_systems.update(p.get("systems", []))
if standalone:
print(f"No standalone emulators found for system(s): {', '.join(system_ids)}",
file=sys.stderr)
else:
print(f"No emulators found for system(s): {', '.join(system_ids)}",
file=sys.stderr)
print(f"Available systems: {', '.join(sorted(all_systems)[:20])}...",
file=sys.stderr)
return None
# Use system-based ZIP name
sys_display = "_".join(
"_".join(w.title() for w in sid.split("-")) for sid in system_ids
)
result = generate_emulator_pack(
matching, emulators_dir, db, bios_dir, output_dir,
standalone, zip_contents,
)
if result:
# Rename to system-based name
new_name = f"{sys_display}_BIOS_Pack.zip"
new_path = os.path.join(output_dir, new_name)
if new_path != result:
os.rename(result, new_path)
result = new_path
return result
def _list_emulators_pack(emulators_dir: str) -> None:
"""Print available emulator profiles for pack generation."""
profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
for name in sorted(profiles):
p = profiles[name]
if p.get("type") in ("alias", "test"):
continue
display = p.get("emulator", name)
ptype = p.get("type", "libretro")
systems = ", ".join(p.get("systems", [])[:3])
more = "..." if len(p.get("systems", [])) > 3 else ""
print(f" {name:30s} {display:40s} [{ptype}] {systems}{more}")
def _list_systems_pack(emulators_dir: str) -> None:
"""Print available system IDs with emulator count."""
profiles = load_emulator_profiles(emulators_dir)
system_emus: dict[str, list[str]] = {}
for name, p in profiles.items():
if p.get("type") in ("alias", "test", "launcher"):
continue
for sys_id in p.get("systems", []):
system_emus.setdefault(sys_id, []).append(name)
for sys_id in sorted(system_emus):
count = len(system_emus[sys_id])
print(f" {sys_id:35s} ({count} emulator{'s' if count > 1 else ''})")
def list_platforms(platforms_dir: str) -> list[str]:
"""List available platform names from YAML files."""
platforms = []
@@ -480,12 +810,16 @@ def main():
parser = argparse.ArgumentParser(description="Generate platform BIOS ZIP packs")
parser.add_argument("--platform", "-p", help="Platform name (e.g., retroarch)")
parser.add_argument("--all", action="store_true", help="Generate packs for all active platforms")
parser.add_argument("--emulator", "-e", help="Emulator profile name(s), comma-separated")
parser.add_argument("--system", "-s", help="System ID(s), comma-separated")
parser.add_argument("--standalone", action="store_true", help="Use standalone mode")
parser.add_argument("--list-emulators", action="store_true", help="List available emulators")
parser.add_argument("--list-systems", action="store_true", help="List available systems")
parser.add_argument("--include-archived", action="store_true", help="Include archived platforms")
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
parser.add_argument("--db", default=DEFAULT_DB_FILE, help="Path to database.json")
parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR)
parser.add_argument("--output-dir", "-o", default=DEFAULT_OUTPUT_DIR)
# --include-extras is now a no-op: core requirements are always included
parser.add_argument("--include-extras", action="store_true",
help="(no-op) Core requirements are always included")
parser.add_argument("--emulators-dir", default="emulators")
@@ -501,7 +835,48 @@ def main():
for p in platforms:
print(p)
return
if args.list_emulators:
_list_emulators_pack(args.emulators_dir)
return
if args.list_systems:
_list_systems_pack(args.emulators_dir)
return
# Mutual exclusion
modes = sum(1 for x in (args.platform, args.all, args.emulator, args.system) if x)
if modes == 0:
parser.error("Specify --platform, --all, --emulator, or --system")
if modes > 1:
parser.error("--platform, --all, --emulator, and --system are mutually exclusive")
if args.standalone and not (args.emulator or args.system):
parser.error("--standalone requires --emulator or --system")
db = load_database(args.db)
zip_contents = build_zip_contents_index(db)
# Emulator mode
if args.emulator:
names = [n.strip() for n in args.emulator.split(",") if n.strip()]
result = generate_emulator_pack(
names, args.emulators_dir, db, args.bios_dir, args.output_dir,
args.standalone, zip_contents,
)
if not result:
sys.exit(1)
return
# System mode
if args.system:
system_ids = [s.strip() for s in args.system.split(",") if s.strip()]
result = generate_system_pack(
system_ids, args.emulators_dir, db, args.bios_dir, args.output_dir,
args.standalone, zip_contents,
)
if not result:
sys.exit(1)
return
# Platform mode (existing)
if args.all:
sys.path.insert(0, os.path.dirname(__file__))
from list_platforms import list_platforms as _list_active
@@ -512,9 +887,6 @@ def main():
parser.error("Specify --platform or --all")
return
db = load_database(args.db)
zip_contents = build_zip_contents_index(db)
data_registry = load_data_dir_registry(args.platforms_dir)
if data_registry and not args.offline:
from refresh_data_dirs import refresh_all, load_registry
@@ -543,7 +915,6 @@ def main():
emu_profiles=emu_profiles,
)
if zip_path and len(group_platforms) > 1:
# Rename ZIP to include all platform names
names = [load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms]
combined_filename = "_".join(n.replace(" ", "") for n in names) + "_BIOS_Pack.zip"
new_path = os.path.join(os.path.dirname(zip_path), combined_filename)

View File

@@ -35,9 +35,10 @@ except ImportError:
sys.path.insert(0, os.path.dirname(__file__))
from common import (
build_zip_contents_index, check_inside_zip, group_identical_platforms,
load_emulator_profiles, load_platform_config, md5sum, md5_composite,
resolve_local_file, resolve_platform_cores,
build_zip_contents_index, check_inside_zip, compute_hashes,
group_identical_platforms, load_data_dir_registry,
load_emulator_profiles, load_platform_config,
md5sum, md5_composite, resolve_local_file, resolve_platform_cores,
)
DEFAULT_DB = "database.json"
@@ -66,17 +67,157 @@ _STATUS_ORDER = {Status.OK: 0, Status.UNTESTED: 1, Status.MISSING: 2}
_SEVERITY_ORDER = {Severity.OK: 0, Severity.INFO: 1, Severity.WARNING: 2, Severity.CRITICAL: 3}
# ---------------------------------------------------------------------------
# Emulator-level validation (size, crc32 checks from emulator profiles)
# ---------------------------------------------------------------------------
def _parse_validation(validation: list | dict | None) -> list[str]:
"""Extract the validation check list from a file's validation field.
Handles both simple list and divergent (core/upstream) dict forms.
For dicts, uses the ``core`` key since RetroArch users run the core.
"""
if validation is None:
return []
if isinstance(validation, list):
return validation
if isinstance(validation, dict):
return validation.get("core", [])
return []
def _build_validation_index(profiles: dict) -> dict[str, dict]:
"""Build per-filename validation rules from emulator profiles.
Returns {filename: {"checks": [str], "size": int|None, "crc32": str|None,
"md5": str|None, "sha1": str|None}}.
When multiple emulators reference the same file, merges checks (union).
Raises ValueError if two profiles declare conflicting values for
the same filename (indicates a profile bug).
"""
index: dict[str, dict] = {}
# Track which emulator set each value, for conflict reporting
sources: dict[str, dict[str, str]] = {}
for emu_name, profile in profiles.items():
if profile.get("type") in ("launcher", "alias"):
continue
for f in profile.get("files", []):
fname = f.get("name", "")
if not fname:
continue
checks = _parse_validation(f.get("validation"))
if not checks:
continue
if fname not in index:
index[fname] = {
"checks": set(), "size": None,
"crc32": None, "md5": None, "sha1": None,
}
sources[fname] = {}
index[fname]["checks"].update(checks)
if "size" in checks and f.get("size") is not None:
new_size = f["size"]
prev_size = index[fname]["size"]
if prev_size is not None and prev_size != new_size:
prev_emu = sources[fname].get("size", "?")
raise ValueError(
f"validation conflict for '{fname}': "
f"size={prev_size} ({prev_emu}) vs size={new_size} ({emu_name})"
)
index[fname]["size"] = new_size
sources[fname]["size"] = emu_name
if "crc32" in checks and f.get("crc32"):
new_crc = f["crc32"].lower()
if new_crc.startswith("0x"):
new_crc = new_crc[2:]
prev_crc = index[fname]["crc32"]
if prev_crc is not None:
norm_prev = prev_crc.lower()
if norm_prev.startswith("0x"):
norm_prev = norm_prev[2:]
if norm_prev != new_crc:
prev_emu = sources[fname].get("crc32", "?")
raise ValueError(
f"validation conflict for '{fname}': "
f"crc32={prev_crc} ({prev_emu}) vs crc32={f['crc32']} ({emu_name})"
)
index[fname]["crc32"] = f["crc32"]
sources[fname]["crc32"] = emu_name
for hash_type in ("md5", "sha1"):
if hash_type in checks and f.get(hash_type):
new_hash = f[hash_type].lower()
prev_hash = index[fname][hash_type]
if prev_hash is not None and prev_hash.lower() != new_hash:
prev_emu = sources[fname].get(hash_type, "?")
raise ValueError(
f"validation conflict for '{fname}': "
f"{hash_type}={prev_hash} ({prev_emu}) vs "
f"{hash_type}={f[hash_type]} ({emu_name})"
)
index[fname][hash_type] = f[hash_type]
sources[fname][hash_type] = emu_name
# Convert sets to sorted lists for determinism
for v in index.values():
v["checks"] = sorted(v["checks"])
return index
def check_file_validation(
local_path: str, filename: str, validation_index: dict[str, dict],
) -> str | None:
"""Check emulator-level validation (size, crc32, md5, sha1) on a resolved file.
Returns None if all checks pass or no validation applies.
Returns a reason string if a check fails.
"""
entry = validation_index.get(filename)
if not entry:
return None
checks = entry["checks"]
if "size" in checks and entry["size"] is not None:
actual_size = os.path.getsize(local_path)
if actual_size != entry["size"]:
return f"size mismatch: expected {entry['size']}, got {actual_size}"
# Hash checks — compute once, reuse
need_hashes = any(
h in checks and entry.get(h) for h in ("crc32", "md5", "sha1")
)
if need_hashes:
hashes = compute_hashes(local_path)
if "crc32" in checks and entry["crc32"]:
expected_crc = entry["crc32"].lower()
if expected_crc.startswith("0x"):
expected_crc = expected_crc[2:]
if hashes["crc32"].lower() != expected_crc:
return f"crc32 mismatch: expected {entry['crc32']}, got {hashes['crc32']}"
if "md5" in checks and entry["md5"]:
if hashes["md5"].lower() != entry["md5"].lower():
return f"md5 mismatch: expected {entry['md5']}, got {hashes['md5']}"
if "sha1" in checks and entry["sha1"]:
if hashes["sha1"].lower() != entry["sha1"].lower():
return f"sha1 mismatch: expected {entry['sha1']}, got {hashes['sha1']}"
return None
# ---------------------------------------------------------------------------
# Verification functions
# ---------------------------------------------------------------------------
def verify_entry_existence(file_entry: dict, local_path: str | None) -> dict:
def verify_entry_existence(
file_entry: dict, local_path: str | None,
validation_index: dict[str, dict] | None = None,
) -> dict:
"""RetroArch verification: path_is_valid() — file exists = OK."""
name = file_entry.get("name", "")
required = file_entry.get("required", True)
if local_path:
return {"name": name, "status": Status.OK, "required": required}
return {"name": name, "status": Status.MISSING, "required": required}
if not local_path:
return {"name": name, "status": Status.MISSING, "required": required}
if validation_index:
reason = check_file_validation(local_path, name, validation_index)
if reason:
return {"name": name, "status": Status.UNTESTED, "required": required,
"path": local_path, "reason": reason}
return {"name": name, "status": Status.OK, "required": required}
def verify_entry_md5(
@@ -325,13 +466,14 @@ def verify_platform(
)
zip_contents = build_zip_contents_index(db) if has_zipped else {}
# Build HLE index from emulator profiles: {filename: True} if any core has HLE for it
# Build HLE + validation indexes from emulator profiles
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir)
hle_index: dict[str, bool] = {}
for profile in profiles.values():
for f in profile.get("files", []):
if f.get("hle_fallback"):
hle_index[f.get("name", "")] = True
validation_index = _build_validation_index(profiles)
# Per-entry results
details = []
@@ -346,9 +488,18 @@ def verify_platform(
file_entry, db, zip_contents,
)
if mode == "existence":
result = verify_entry_existence(file_entry, local_path)
result = verify_entry_existence(
file_entry, local_path, validation_index,
)
else:
result = verify_entry_md5(file_entry, local_path, resolve_status)
# Apply emulator-level validation on top of MD5 check
if result["status"] == Status.OK and local_path and validation_index:
fname = file_entry.get("name", "")
reason = check_file_validation(local_path, fname, validation_index)
if reason:
result["status"] = Status.UNTESTED
result["reason"] = reason
result["system"] = sys_id
result["hle_fallback"] = hle_index.get(file_entry.get("name", ""), False)
details.append(result)
@@ -504,10 +655,328 @@ def print_platform_result(result: dict, group: list[str]) -> None:
print(f" {ex['emulator']}{ex['detail']} [{ex['reason']}]")
# ---------------------------------------------------------------------------
# Emulator/system mode verification
# ---------------------------------------------------------------------------
def _filter_files_by_mode(files: list[dict], standalone: bool) -> list[dict]:
"""Filter file entries by libretro/standalone mode."""
result = []
for f in files:
fmode = f.get("mode", "")
if standalone and fmode == "libretro":
continue
if not standalone and fmode == "standalone":
continue
result.append(f)
return result
def _effective_validation_label(details: list[dict], validation_index: dict) -> str:
"""Determine the bracket label for the report.
Returns the union of all check types used, e.g. [crc32+existence+size].
"""
all_checks: set[str] = set()
has_files = False
for d in details:
fname = d.get("name", "")
if d.get("note"):
continue # skip informational entries (empty profiles)
has_files = True
entry = validation_index.get(fname)
if entry:
all_checks.update(entry["checks"])
else:
all_checks.add("existence")
if not has_files:
return "existence"
return "+".join(sorted(all_checks))
def verify_emulator(
profile_names: list[str],
emulators_dir: str,
db: dict,
standalone: bool = False,
) -> dict:
"""Verify files for specific emulator profiles."""
profiles = load_emulator_profiles(emulators_dir)
zip_contents = build_zip_contents_index(db)
# Also load aliases for redirect messages
all_profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
# Resolve profile names, reject alias/launcher
selected: list[tuple[str, dict]] = []
for name in profile_names:
if name not in all_profiles:
available = sorted(k for k, v in all_profiles.items()
if v.get("type") not in ("alias", "test"))
print(f"Error: emulator '{name}' not found", file=sys.stderr)
print(f"Available: {', '.join(available[:10])}...", file=sys.stderr)
sys.exit(1)
p = all_profiles[name]
if p.get("type") == "alias":
alias_of = p.get("alias_of", "?")
print(f"Error: {name} is an alias of {alias_of} — use --emulator {alias_of}",
file=sys.stderr)
sys.exit(1)
if p.get("type") == "launcher":
print(f"Error: {name} is a launcher — use the emulator it launches",
file=sys.stderr)
sys.exit(1)
# Check standalone capability
ptype = p.get("type", "libretro")
if standalone and "standalone" not in ptype:
print(f"Error: {name} ({ptype}) does not support --standalone",
file=sys.stderr)
sys.exit(1)
selected.append((name, p))
# Build validation index from selected profiles only
selected_profiles = {n: p for n, p in selected}
validation_index = _build_validation_index(selected_profiles)
data_registry = load_data_dir_registry(
os.path.join(os.path.dirname(__file__), "..", "platforms")
)
details = []
file_status: dict[str, str] = {}
file_severity: dict[str, str] = {}
data_dir_notices: list[str] = []
for emu_name, profile in selected:
files = _filter_files_by_mode(profile.get("files", []), standalone)
# Check data directories (only notice if not cached)
for dd in profile.get("data_directories", []):
ref = dd.get("ref", "")
if not ref:
continue
if data_registry and ref in data_registry:
cache_path = data_registry[ref].get("local_cache", "")
if cache_path and os.path.isdir(cache_path):
continue # cached, no notice needed
data_dir_notices.append(ref)
if not files:
details.append({
"name": f"({emu_name})", "status": Status.OK,
"required": False, "system": "",
"note": f"No files needed for {profile.get('emulator', emu_name)}",
})
continue
# Verify archives as units (e.g., neogeo.zip, aes.zip)
seen_archives: set[str] = set()
for file_entry in files:
archive = file_entry.get("archive")
if archive and archive not in seen_archives:
seen_archives.add(archive)
archive_entry = {"name": archive}
local_path, _ = resolve_local_file(archive_entry, db, zip_contents)
required = any(
f.get("archive") == archive and f.get("required", True)
for f in files
)
if local_path:
result = {"name": archive, "status": Status.OK,
"required": required, "path": local_path}
else:
result = {"name": archive, "status": Status.MISSING,
"required": required}
result["system"] = file_entry.get("system", "")
result["hle_fallback"] = False
details.append(result)
dest = archive
cur = result["status"]
prev = file_status.get(dest)
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
file_status[dest] = cur
sev = compute_severity(cur, required, "existence", False)
prev_sev = file_severity.get(dest)
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(prev_sev, 0):
file_severity[dest] = sev
for file_entry in files:
# Skip archived files (verified as archive units above)
if file_entry.get("archive"):
continue
dest_hint = file_entry.get("path", "")
local_path, resolve_status = resolve_local_file(
file_entry, db, zip_contents, dest_hint=dest_hint,
)
name = file_entry.get("name", "")
required = file_entry.get("required", True)
hle = file_entry.get("hle_fallback", False)
if not local_path:
result = {"name": name, "status": Status.MISSING, "required": required}
else:
# Apply emulator validation
reason = check_file_validation(local_path, name, validation_index)
if reason:
result = {"name": name, "status": Status.UNTESTED,
"required": required, "path": local_path,
"reason": reason}
else:
result = {"name": name, "status": Status.OK,
"required": required, "path": local_path}
result["system"] = file_entry.get("system", "")
result["hle_fallback"] = hle
details.append(result)
# Aggregate by destination (path if available, else name)
dest = file_entry.get("path", "") or name
cur = result["status"]
prev = file_status.get(dest)
if prev is None or _STATUS_ORDER.get(cur, 0) > _STATUS_ORDER.get(prev, 0):
file_status[dest] = cur
sev = compute_severity(cur, required, "existence", hle)
prev_sev = file_severity.get(dest)
if prev_sev is None or _SEVERITY_ORDER.get(sev, 0) > _SEVERITY_ORDER.get(prev_sev, 0):
file_severity[dest] = sev
counts = {Severity.OK: 0, Severity.INFO: 0, Severity.WARNING: 0, Severity.CRITICAL: 0}
for s in file_severity.values():
counts[s] = counts.get(s, 0) + 1
status_counts: dict[str, int] = {}
for s in file_status.values():
status_counts[s] = status_counts.get(s, 0) + 1
label = _effective_validation_label(details, validation_index)
return {
"emulators": [n for n, _ in selected],
"verification_mode": label,
"total_files": len(file_status),
"severity_counts": counts,
"status_counts": status_counts,
"details": details,
"data_dir_notices": sorted(set(data_dir_notices)),
}
def verify_system(
system_ids: list[str],
emulators_dir: str,
db: dict,
standalone: bool = False,
) -> dict:
"""Verify files for all emulators supporting given system IDs."""
profiles = load_emulator_profiles(emulators_dir)
matching = []
for name, profile in sorted(profiles.items()):
if profile.get("type") in ("launcher", "alias", "test"):
continue
emu_systems = set(profile.get("systems", []))
if emu_systems & set(system_ids):
ptype = profile.get("type", "libretro")
if standalone and "standalone" not in ptype:
continue # skip non-standalone in standalone mode
matching.append(name)
if not matching:
all_systems: set[str] = set()
for p in profiles.values():
all_systems.update(p.get("systems", []))
if standalone:
print(f"No standalone emulators found for system(s): {', '.join(system_ids)}",
file=sys.stderr)
else:
print(f"No emulators found for system(s): {', '.join(system_ids)}",
file=sys.stderr)
print(f"Available systems: {', '.join(sorted(all_systems)[:20])}...",
file=sys.stderr)
sys.exit(1)
return verify_emulator(matching, emulators_dir, db, standalone)
def print_emulator_result(result: dict) -> None:
"""Print verification result for emulator/system mode."""
label = " + ".join(result["emulators"])
mode = result["verification_mode"]
total = result["total_files"]
c = result["severity_counts"]
ok_count = c[Severity.OK]
sc = result.get("status_counts", {})
untested = sc.get(Status.UNTESTED, 0)
missing = sc.get(Status.MISSING, 0)
parts = [f"{ok_count}/{total} OK"]
if untested:
parts.append(f"{untested} untested")
if missing:
parts.append(f"{missing} missing")
print(f"{label}: {', '.join(parts)} [{mode}]")
seen = set()
for d in result["details"]:
if d["status"] == Status.UNTESTED:
if d["name"] in seen:
continue
seen.add(d["name"])
req = "required" if d.get("required", True) else "optional"
hle = ", HLE available" if d.get("hle_fallback") else ""
reason = d.get("reason", "")
print(f" UNTESTED ({req}{hle}): {d['name']}{reason}")
for d in result["details"]:
if d["status"] == Status.MISSING:
if d["name"] in seen:
continue
seen.add(d["name"])
req = "required" if d.get("required", True) else "optional"
hle = ", HLE available" if d.get("hle_fallback") else ""
print(f" MISSING ({req}{hle}): {d['name']}")
for d in result["details"]:
if d.get("note"):
print(f" {d['note']}")
for ref in result.get("data_dir_notices", []):
print(f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)")
def _list_emulators(emulators_dir: str) -> None:
"""Print available emulator profiles."""
profiles = load_emulator_profiles(emulators_dir)
for name in sorted(profiles):
p = profiles[name]
if p.get("type") in ("alias", "test"):
continue
display = p.get("emulator", name)
ptype = p.get("type", "libretro")
systems = ", ".join(p.get("systems", [])[:3])
more = "..." if len(p.get("systems", [])) > 3 else ""
print(f" {name:30s} {display:40s} [{ptype}] {systems}{more}")
def _list_systems(emulators_dir: str) -> None:
"""Print available system IDs with emulator count."""
profiles = load_emulator_profiles(emulators_dir)
system_emus: dict[str, list[str]] = {}
for name, p in profiles.items():
if p.get("type") in ("alias", "test", "launcher"):
continue
for sys_id in p.get("systems", []):
system_emus.setdefault(sys_id, []).append(name)
for sys_id in sorted(system_emus):
count = len(system_emus[sys_id])
print(f" {sys_id:35s} ({count} emulator{'s' if count > 1 else ''})")
def main():
parser = argparse.ArgumentParser(description="Platform-native BIOS verification")
parser.add_argument("--platform", "-p", help="Platform name")
parser.add_argument("--all", action="store_true", help="Verify all active platforms")
parser.add_argument("--emulator", "-e", help="Emulator profile name(s), comma-separated")
parser.add_argument("--system", "-s", help="System ID(s), comma-separated")
parser.add_argument("--standalone", action="store_true", help="Use standalone mode")
parser.add_argument("--list-emulators", action="store_true", help="List available emulators")
parser.add_argument("--list-systems", action="store_true", help="List available systems")
parser.add_argument("--include-archived", action="store_true")
parser.add_argument("--db", default=DEFAULT_DB)
parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR)
@@ -515,9 +984,48 @@ def main():
parser.add_argument("--json", action="store_true", help="JSON output")
args = parser.parse_args()
if args.list_emulators:
_list_emulators(args.emulators_dir)
return
if args.list_systems:
_list_systems(args.emulators_dir)
return
# Mutual exclusion
modes = sum(1 for x in (args.platform, args.all, args.emulator, args.system) if x)
if modes == 0:
parser.error("Specify --platform, --all, --emulator, or --system")
if modes > 1:
parser.error("--platform, --all, --emulator, and --system are mutually exclusive")
if args.standalone and not (args.emulator or args.system):
parser.error("--standalone requires --emulator or --system")
with open(args.db) as f:
db = json.load(f)
# Emulator mode
if args.emulator:
names = [n.strip() for n in args.emulator.split(",") if n.strip()]
result = verify_emulator(names, args.emulators_dir, db, args.standalone)
if args.json:
result["details"] = [d for d in result["details"] if d["status"] != Status.OK]
print(json.dumps(result, indent=2))
else:
print_emulator_result(result)
return
# System mode
if args.system:
system_ids = [s.strip() for s in args.system.split(",") if s.strip()]
result = verify_system(system_ids, args.emulators_dir, db, args.standalone)
if args.json:
result["details"] = [d for d in result["details"] if d["status"] != Status.OK]
print(json.dumps(result, indent=2))
else:
print_emulator_result(result)
return
# Platform mode (existing)
if args.all:
from list_platforms import list_platforms as _list_platforms
platforms = _list_platforms(include_archived=args.include_archived)

View File

@@ -33,7 +33,11 @@ from common import (
load_emulator_profiles, load_platform_config, md5_composite, md5sum,
resolve_local_file, resolve_platform_cores,
)
from verify import Severity, Status, verify_platform, find_undeclared_files, find_exclusion_notes
from verify import (
Severity, Status, verify_platform, find_undeclared_files, find_exclusion_notes,
_build_validation_index, check_file_validation, verify_emulator,
_filter_files_by_mode, _effective_validation_label,
)
def _h(data: bytes) -> dict:
@@ -71,6 +75,13 @@ class TestE2E(unittest.TestCase):
self._make_file("no_md5.bin", b"NO_MD5_CHECK")
self._make_file("truncated.bin", b"BATOCERA_TRUNCATED")
self._make_file("alias_target.bin", b"ALIAS_FILE_DATA")
self._make_file("leading_zero_crc.bin", b"LEADING_ZERO_CRC_12") # crc32=0179e92e
# Regional variant files (same name, different content, in subdirs)
os.makedirs(os.path.join(self.bios_dir, "TestConsole", "USA"), exist_ok=True)
os.makedirs(os.path.join(self.bios_dir, "TestConsole", "EUR"), exist_ok=True)
self._make_file("BIOS.bin", b"BIOS_USA_CONTENT", subdir="TestConsole/USA")
self._make_file("BIOS.bin", b"BIOS_EUR_CONTENT", subdir="TestConsole/EUR")
# .variants/ file (should be deprioritized)
variants_dir = os.path.join(self.bios_dir, ".variants")
@@ -149,9 +160,16 @@ class TestE2E(unittest.TestCase):
# Add alias name to by_name
alias_sha1 = self.files["alias_target.bin"]["sha1"]
by_name.setdefault("alias_alt.bin", []).append(alias_sha1)
# Build by_path_suffix for regional variant resolution
by_path_suffix = {}
for key, info in self.files.items():
if "/" in key:
# key is subdir/name, suffix is the subdir path
by_path_suffix.setdefault(key, []).append(info["sha1"])
return {
"files": files_db,
"indexes": {"by_md5": by_md5, "by_name": by_name, "by_crc32": {}},
"indexes": {"by_md5": by_md5, "by_name": by_name, "by_crc32": {},
"by_path_suffix": by_path_suffix},
}
# ---------------------------------------------------------------
@@ -322,6 +340,41 @@ class TestE2E(unittest.TestCase):
with open(os.path.join(self.emulators_dir, "test_emu_dd.yml"), "w") as fh:
yaml.dump(emu_dd, fh)
# Emulator with validation checks (size, crc32)
emu_val = {
"emulator": "TestValidation",
"type": "libretro",
"systems": ["console-a", "sys-md5"],
"files": [
# Size validation — correct size (16 bytes = len(b"PRESENT_REQUIRED"))
{"name": "present_req.bin", "required": True,
"validation": ["size"], "size": 16},
# Size validation — wrong expected size
{"name": "present_opt.bin", "required": False,
"validation": ["size"], "size": 9999},
# CRC32 validation — correct crc32
{"name": "correct_hash.bin", "required": True,
"validation": ["crc32"], "crc32": "91d0b1d3"},
# CRC32 validation — wrong crc32
{"name": "no_md5.bin", "required": False,
"validation": ["crc32"], "crc32": "deadbeef"},
# CRC32 starting with '0' (regression: lstrip("0x") bug)
{"name": "leading_zero_crc.bin", "required": True,
"validation": ["crc32"], "crc32": "0179e92e"},
# MD5 validation — correct md5
{"name": "correct_hash.bin", "required": True,
"validation": ["md5"], "md5": "4a8db431e3b1a1acacec60e3424c4ce8"},
# SHA1 validation — correct sha1
{"name": "correct_hash.bin", "required": True,
"validation": ["sha1"], "sha1": "a2ab6c95c5bbd191b9e87e8f4e85205a47be5764"},
# MD5 validation — wrong md5
{"name": "alias_target.bin", "required": False,
"validation": ["md5"], "md5": "0000000000000000000000000000dead"},
],
}
with open(os.path.join(self.emulators_dir, "test_validation.yml"), "w") as fh:
yaml.dump(emu_val, fh)
# ---------------------------------------------------------------
# THE TEST — one method per feature area, all using same fixtures
# ---------------------------------------------------------------
@@ -641,5 +694,276 @@ class TestE2E(unittest.TestCase):
self.assertIn("DeSmuME 2015", emu_names)
def test_70_validation_index_built(self):
"""Validation index extracts checks from emulator profiles."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
self.assertIn("present_req.bin", index)
self.assertIn("size", index["present_req.bin"]["checks"])
self.assertEqual(index["present_req.bin"]["size"], 16)
self.assertIn("correct_hash.bin", index)
self.assertIn("crc32", index["correct_hash.bin"]["checks"])
def test_71_validation_size_pass(self):
"""File with correct size passes validation."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
path = self.files["present_req.bin"]["path"]
reason = check_file_validation(path, "present_req.bin", index)
self.assertIsNone(reason)
def test_72_validation_size_fail(self):
"""File with wrong size fails validation."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
path = self.files["present_opt.bin"]["path"]
reason = check_file_validation(path, "present_opt.bin", index)
self.assertIsNotNone(reason)
self.assertIn("size mismatch", reason)
def test_73_validation_crc32_pass(self):
"""File with correct CRC32 passes validation."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
path = self.files["correct_hash.bin"]["path"]
reason = check_file_validation(path, "correct_hash.bin", index)
self.assertIsNone(reason)
def test_74_validation_crc32_fail(self):
"""File with wrong CRC32 fails validation."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
path = self.files["no_md5.bin"]["path"]
reason = check_file_validation(path, "no_md5.bin", index)
self.assertIsNotNone(reason)
self.assertIn("crc32 mismatch", reason)
def test_75_validation_applied_in_existence_mode(self):
"""Existence platform downgrades OK to UNTESTED when validation fails."""
config = load_platform_config("test_existence", self.platforms_dir)
profiles = load_emulator_profiles(self.emulators_dir)
result = verify_platform(config, self.db, self.emulators_dir, profiles)
# present_opt.bin exists but has wrong expected size → UNTESTED
for d in result["details"]:
if d["name"] == "present_opt.bin":
self.assertEqual(d["status"], Status.UNTESTED)
self.assertIn("size mismatch", d.get("reason", ""))
break
else:
self.fail("present_opt.bin not found in details")
def test_77_validation_crc32_leading_zero(self):
"""CRC32 starting with '0' must not be truncated (lstrip regression)."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
path = self.files["leading_zero_crc.bin"]["path"]
reason = check_file_validation(path, "leading_zero_crc.bin", index)
self.assertIsNone(reason)
def test_78_validation_conflict_raises(self):
"""Conflicting size/crc32 from two profiles raises ValueError."""
profiles = {
"emu_a": {
"type": "libretro", "files": [
{"name": "shared.bin", "validation": ["size"], "size": 512},
],
},
"emu_b": {
"type": "libretro", "files": [
{"name": "shared.bin", "validation": ["size"], "size": 1024},
],
},
}
with self.assertRaises(ValueError) as ctx:
_build_validation_index(profiles)
self.assertIn("validation conflict", str(ctx.exception))
self.assertIn("shared.bin", str(ctx.exception))
def test_79_validation_md5_pass(self):
"""File with correct MD5 passes validation."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
path = self.files["correct_hash.bin"]["path"]
reason = check_file_validation(path, "correct_hash.bin", index)
self.assertIsNone(reason)
def test_80_validation_md5_fail(self):
"""File with wrong MD5 fails validation."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
path = self.files["alias_target.bin"]["path"]
reason = check_file_validation(path, "alias_target.bin", index)
self.assertIsNotNone(reason)
self.assertIn("md5 mismatch", reason)
def test_81_validation_index_has_md5_sha1(self):
"""Validation index stores md5 and sha1 when declared."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
self.assertIn("md5", index["correct_hash.bin"]["checks"])
self.assertIn("sha1", index["correct_hash.bin"]["checks"])
self.assertIsNotNone(index["correct_hash.bin"]["md5"])
self.assertIsNotNone(index["correct_hash.bin"]["sha1"])
def test_76_validation_no_effect_when_no_field(self):
"""Files without validation field are unaffected."""
profiles = load_emulator_profiles(self.emulators_dir)
index = _build_validation_index(profiles)
# wrong_hash.bin has no validation in any profile
path = self.files["wrong_hash.bin"]["path"]
reason = check_file_validation(path, "wrong_hash.bin", index)
self.assertIsNone(reason)
# ---------------------------------------------------------------
# Emulator/system mode verification
# ---------------------------------------------------------------
def test_90_verify_emulator_basic(self):
"""verify_emulator returns correct counts for a profile with mixed present/missing."""
result = verify_emulator(["test_emu"], self.emulators_dir, self.db)
self.assertIn("test_emu", result["emulators"])
# present_req.bin and alias_target.bin are present, others missing
self.assertGreater(result["total_files"], 0)
self.assertGreater(result["severity_counts"][Severity.OK], 0)
def test_91_verify_emulator_standalone_filters(self):
"""Standalone mode includes mode:standalone files, excludes mode:libretro."""
result_lr = verify_emulator(["test_emu"], self.emulators_dir, self.db, standalone=False)
result_sa = verify_emulator(["test_emu"], self.emulators_dir, self.db, standalone=True)
lr_names = {d["name"] for d in result_lr["details"]}
sa_names = {d["name"] for d in result_sa["details"]}
# standalone_only.bin should be in standalone, not libretro
self.assertNotIn("standalone_only.bin", lr_names)
self.assertIn("standalone_only.bin", sa_names)
def test_102_resolve_dest_hint_disambiguates(self):
"""dest_hint resolves regional variants with same name to distinct files."""
usa_path, usa_status = resolve_local_file(
{"name": "BIOS.bin"}, self.db, dest_hint="TestConsole/USA/BIOS.bin",
)
eur_path, eur_status = resolve_local_file(
{"name": "BIOS.bin"}, self.db, dest_hint="TestConsole/EUR/BIOS.bin",
)
self.assertIsNotNone(usa_path)
self.assertIsNotNone(eur_path)
self.assertEqual(usa_status, "exact")
self.assertEqual(eur_status, "exact")
# Must be DIFFERENT files
self.assertNotEqual(usa_path, eur_path)
# Verify content
with open(usa_path, "rb") as f:
self.assertEqual(f.read(), b"BIOS_USA_CONTENT")
with open(eur_path, "rb") as f:
self.assertEqual(f.read(), b"BIOS_EUR_CONTENT")
def test_103_resolve_dest_hint_fallback_to_name(self):
"""Without dest_hint, falls back to by_name (first candidate)."""
path, status = resolve_local_file({"name": "BIOS.bin"}, self.db)
self.assertIsNotNone(path)
# Still finds something (first candidate by name)
def test_92_verify_emulator_libretro_only_rejects_standalone(self):
"""Libretro-only profile rejects --standalone."""
with self.assertRaises(SystemExit):
verify_emulator(["test_hle"], self.emulators_dir, self.db, standalone=True)
def test_92b_verify_emulator_game_type_rejects_standalone(self):
"""Game-type profile rejects --standalone."""
game = {"emulator": "TestGame", "type": "game", "systems": ["console-a"], "files": []}
with open(os.path.join(self.emulators_dir, "test_game.yml"), "w") as fh:
yaml.dump(game, fh)
with self.assertRaises(SystemExit):
verify_emulator(["test_game"], self.emulators_dir, self.db, standalone=True)
def test_93_verify_emulator_alias_rejected(self):
"""Alias profile produces error with redirect message."""
with self.assertRaises(SystemExit):
verify_emulator(["test_alias"], self.emulators_dir, self.db)
def test_94_verify_emulator_launcher_rejected(self):
"""Launcher profile produces error."""
with self.assertRaises(SystemExit):
verify_emulator(["test_launcher"], self.emulators_dir, self.db)
def test_95_verify_emulator_validation_applied(self):
"""Emulator mode applies validation checks as primary verification."""
result = verify_emulator(["test_validation"], self.emulators_dir, self.db)
# present_opt.bin has wrong size → UNTESTED
for d in result["details"]:
if d["name"] == "present_opt.bin":
self.assertEqual(d["status"], Status.UNTESTED)
self.assertIn("size mismatch", d.get("reason", ""))
break
else:
self.fail("present_opt.bin not found in details")
def test_96_verify_emulator_multi(self):
"""Multi-emulator verify aggregates files."""
result = verify_emulator(
["test_emu", "test_hle"], self.emulators_dir, self.db,
)
self.assertEqual(len(result["emulators"]), 2)
all_names = {d["name"] for d in result["details"]}
# Files from both profiles
self.assertIn("present_req.bin", all_names)
self.assertIn("hle_missing.bin", all_names)
def test_97_verify_emulator_data_dir_notice(self):
"""Emulator with data_directories reports notice."""
result = verify_emulator(["test_emu"], self.emulators_dir, self.db)
self.assertIn("test-data-dir", result.get("data_dir_notices", []))
def test_98_verify_emulator_validation_label(self):
"""Validation label reflects the checks used."""
result = verify_emulator(["test_validation"], self.emulators_dir, self.db)
# test_validation has crc32, md5, sha1, size → all listed
self.assertEqual(result["verification_mode"], "crc32+md5+sha1+size")
def test_99_filter_files_by_mode(self):
"""_filter_files_by_mode correctly filters standalone/libretro."""
files = [
{"name": "a.bin"}, # no mode → both
{"name": "b.bin", "mode": "libretro"}, # libretro only
{"name": "c.bin", "mode": "standalone"}, # standalone only
{"name": "d.bin", "mode": "both"}, # explicit both
]
lr = _filter_files_by_mode(files, standalone=False)
sa = _filter_files_by_mode(files, standalone=True)
lr_names = {f["name"] for f in lr}
sa_names = {f["name"] for f in sa}
self.assertEqual(lr_names, {"a.bin", "b.bin", "d.bin"})
self.assertEqual(sa_names, {"a.bin", "c.bin", "d.bin"})
def test_100_verify_emulator_empty_profile(self):
"""Profile with files:[] produces note, not error."""
empty = {
"emulator": "TestEmpty", "type": "libretro",
"systems": ["console-a"], "files": [],
"exclusion_note": "Code never loads BIOS",
}
with open(os.path.join(self.emulators_dir, "test_empty.yml"), "w") as fh:
yaml.dump(empty, fh)
result = verify_emulator(["test_empty"], self.emulators_dir, self.db)
# Should have a note entry, not crash
self.assertEqual(result["total_files"], 0)
notes = [d for d in result["details"] if d.get("note")]
self.assertTrue(len(notes) > 0)
def test_101_verify_emulator_severity_missing_required(self):
"""Missing required file in emulator mode → WARNING severity."""
result = verify_emulator(["test_emu"], self.emulators_dir, self.db)
# undeclared_req.bin is required and missing
for d in result["details"]:
if d["name"] == "undeclared_req.bin":
self.assertEqual(d["status"], Status.MISSING)
self.assertTrue(d["required"])
break
else:
self.fail("undeclared_req.bin not found")
# Severity should be WARNING (existence mode base)
self.assertGreater(result["severity_counts"][Severity.WARNING], 0)
if __name__ == "__main__":
unittest.main()