mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-04-13 12:22:33 -05:00
refactor: harden codebase and remove unicode artifacts
- fix urllib.parse.quote import (was urllib.request.quote) - add operator precedence parens in generate_pack dedup check - narrow bare except to specific types in batocera target scraper - cache load_platform_config and build_zip_contents_index results - add selective algorithm support to compute_hashes - atomic write for fetch_large_file (tmp + rename) - add response size limit to base scraper fetch - extract build_target_cores_cache to common.py (dedup verify/pack) - hoist _build_supplemental_index out of per-platform loop - migrate function-attribute caches to module-level dicts - add @abstractmethod to BaseTargetScraper.fetch_targets - remove backward-compat re-exports from common.py - replace em-dashes and unicode arrows with ASCII equivalents - remove decorative section dividers and obvious comments
This commit is contained in:
Abdessamad Derraz
@@ -471,7 +471,7 @@ def resolve_local_file(
|
||||
if valid:
|
||||
primary = [p for p, _ in valid if "/.variants/" not in p]
|
||||
return (primary[0] if primary else valid[0][0]), "hash_mismatch"
|
||||
# No candidate contains the zipped_file — fall through to step 5
|
||||
# No candidate contains the zipped_file -fall through to step 5
|
||||
else:
|
||||
primary = [p for p, _ in candidates if "/.variants/" not in p]
|
||||
return (primary[0] if primary else candidates[0][0]), "hash_mismatch"
|
||||
@@ -550,7 +550,7 @@ def _get_mame_clone_map() -> dict[str, str]:
|
||||
|
||||
|
||||
def check_inside_zip(container: str, file_name: str, expected_md5: str) -> str:
|
||||
"""Check a ROM inside a ZIP — replicates Batocera checkInsideZip().
|
||||
"""Check a ROM inside a ZIP -replicates Batocera checkInsideZip().
|
||||
|
||||
Returns "ok", "untested", "not_in_zip", or "error".
|
||||
"""
|
||||
@@ -765,7 +765,7 @@ MANUFACTURER_PREFIXES = (
|
||||
"snk-", "panasonic-", "nec-", "epoch-", "mattel-", "fairchild-",
|
||||
"hartung-", "tiger-", "magnavox-", "philips-", "bandai-", "casio-",
|
||||
"coleco-", "commodore-", "sharp-", "sinclair-", "atari-", "sammy-",
|
||||
"gce-", "texas-instruments-",
|
||||
"gce-", "interton-", "texas-instruments-",
|
||||
)
|
||||
|
||||
|
||||
@@ -869,20 +869,20 @@ def filter_systems_by_target(
|
||||
plat_cores_here = norm_plat_system_cores.get(norm_key, set())
|
||||
|
||||
if not all_cores and not plat_cores_here:
|
||||
# No profile maps to this system — keep it
|
||||
# No profile maps to this system -keep it
|
||||
filtered[sys_id] = sys_data
|
||||
elif all_cores & expanded_target:
|
||||
# At least one core is on the target
|
||||
filtered[sys_id] = sys_data
|
||||
elif not plat_cores_here:
|
||||
# Platform resolution didn't find cores for this system — keep it
|
||||
# Platform resolution didn't find cores for this system -keep it
|
||||
filtered[sys_id] = sys_data
|
||||
# else: known cores exist but none are on the target — exclude
|
||||
# else: known cores exist but none are on the target -exclude
|
||||
return filtered
|
||||
|
||||
|
||||
|
||||
# Validation and mode filtering — extracted to validation.py for SoC.
|
||||
# Validation and mode filtering -extracted to validation.py for SoC.
|
||||
# Re-exported below for backward compatibility.
|
||||
|
||||
|
||||
|
||||
@@ -23,9 +23,7 @@ from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Key file parsing (keys.txt / aes_keys.txt format)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def parse_keys_file(path: str | Path) -> dict[str, dict[str, bytes]]:
|
||||
"""Parse a 3DS keys file with :AES, :RSA, :ECC sections.
|
||||
@@ -67,9 +65,7 @@ def find_keys_file(bios_dir: str | Path) -> Path | None:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pure Python RSA-2048 PKCS1v15 SHA256 verification (zero dependencies)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _rsa_verify_pkcs1v15_sha256(
|
||||
message: bytes,
|
||||
@@ -79,7 +75,7 @@ def _rsa_verify_pkcs1v15_sha256(
|
||||
) -> bool:
|
||||
"""Verify RSA-2048 PKCS#1 v1.5 with SHA-256.
|
||||
|
||||
Pure Python — uses Python's native int for modular exponentiation.
|
||||
Pure Python -uses Python's native int for modular exponentiation.
|
||||
Reproduces CryptoPP::RSASS<PKCS1v15, SHA256>::Verifier.
|
||||
"""
|
||||
n = int.from_bytes(modulus, "big")
|
||||
@@ -124,9 +120,7 @@ def _rsa_verify_pkcs1v15_sha256(
|
||||
return em == expected_em
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AES-128-CBC decryption (with fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
|
||||
"""Decrypt AES-128-CBC without padding."""
|
||||
@@ -166,9 +160,7 @@ def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# File verification functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def verify_secure_info_a(
|
||||
filepath: str | Path,
|
||||
@@ -347,7 +339,7 @@ def verify_otp(
|
||||
if computed_hash != stored_hash:
|
||||
return False, "SHA-256 hash mismatch (OTP corrupted)"
|
||||
|
||||
# --- ECC certificate verification (sect233r1) ---
|
||||
# ECC certificate verification (sect233r1)
|
||||
ecc_keys = keys.get("ECC", {})
|
||||
root_public_xy = ecc_keys.get("rootPublicXY")
|
||||
if not root_public_xy or len(root_public_xy) != 60:
|
||||
@@ -414,9 +406,7 @@ def verify_otp(
|
||||
return False, "decrypted, magic+SHA256 valid, but ECC cert signature invalid"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unified verification interface for verify.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Map from (filename, validation_type) to verification function
|
||||
_CRYPTO_VERIFIERS: dict[str, Callable] = {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Deduplicate bios/ directory — keep one canonical file per unique content.
|
||||
"""Deduplicate bios/ directory -keep one canonical file per unique content.
|
||||
|
||||
Usage:
|
||||
python scripts/dedup.py [--dry-run] [--bios-dir bios]
|
||||
@@ -11,7 +11,7 @@ Two types of deduplication:
|
||||
|
||||
2. MAME DEVICE CLONES: Different filenames with identical content in the same
|
||||
MAME directory (e.g., bbc_m87.zip and bbc_24bbc.zip are identical ZIPs).
|
||||
These are NOT aliases — MAME loads each by its unique name. Instead of
|
||||
These are NOT aliases -MAME loads each by its unique name. Instead of
|
||||
deleting, we create a _mame_clones.json mapping so generate_pack.py can
|
||||
pack all names from a single canonical file.
|
||||
|
||||
@@ -94,7 +94,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
|
||||
if len(paths) <= 1:
|
||||
continue
|
||||
|
||||
# Separate by filename — same name = true duplicate, different name = clone
|
||||
# Separate by filename -same name = true duplicate, different name = clone
|
||||
by_name: dict[str, list[str]] = defaultdict(list)
|
||||
for p in paths:
|
||||
by_name[os.path.basename(p)].append(p)
|
||||
@@ -106,7 +106,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
|
||||
name_paths.sort(key=path_priority)
|
||||
true_dupes_to_remove.extend(name_paths[1:])
|
||||
|
||||
# Different filenames, same content — need special handling
|
||||
# Different filenames, same content -need special handling
|
||||
unique_names = sorted(by_name.keys())
|
||||
if len(unique_names) > 1:
|
||||
# Check if these are all in MAME/Arcade dirs AND all ZIPs
|
||||
@@ -133,7 +133,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
|
||||
true_dupes_to_remove.append(p)
|
||||
else:
|
||||
# Non-MAME different names (e.g., 64DD_IPL_US.n64 vs IPL_USA.n64)
|
||||
# Keep ALL — each name may be needed by a different emulator
|
||||
# Keep ALL -each name may be needed by a different emulator
|
||||
# Only remove true duplicates (same name in multiple dirs)
|
||||
pass
|
||||
|
||||
@@ -143,7 +143,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
|
||||
# Find the best canonical across all paths
|
||||
all_paths = [p for p in paths if p not in true_dupes_to_remove]
|
||||
if not all_paths:
|
||||
# All copies were marked for removal — keep the best one
|
||||
# All copies were marked for removal -keep the best one
|
||||
all_paths_sorted = sorted(paths, key=path_priority)
|
||||
all_paths = [all_paths_sorted[0]]
|
||||
true_dupes_to_remove = [p for p in paths if p != all_paths[0]]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Deterministic ZIP builder for MAME BIOS archives.
|
||||
|
||||
Creates byte-identical ZIP files from individual ROM atoms, enabling:
|
||||
- Reproducible builds: same ROMs → same ZIP hash, always
|
||||
- Reproducible builds: same ROMs -> same ZIP hash, always
|
||||
- Version-agnostic assembly: build neogeo.zip for any MAME version
|
||||
- Deduplication: store ROM atoms once, assemble any ZIP on demand
|
||||
|
||||
|
||||
@@ -141,8 +141,8 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
|
||||
def _path_suffix(rel_path: str) -> str:
|
||||
"""Extract the path suffix after bios/Manufacturer/Console/.
|
||||
|
||||
bios/Nintendo/GameCube/GC/USA/IPL.bin → GC/USA/IPL.bin
|
||||
bios/Sony/PlayStation/scph5501.bin → scph5501.bin
|
||||
bios/Nintendo/GameCube/GC/USA/IPL.bin -> GC/USA/IPL.bin
|
||||
bios/Sony/PlayStation/scph5501.bin -> scph5501.bin
|
||||
"""
|
||||
parts = rel_path.replace("\\", "/").split("/")
|
||||
# Skip: bios / Manufacturer / Console (3 segments)
|
||||
|
||||
@@ -248,7 +248,7 @@ def resolve_file(file_entry: dict, db: dict, bios_dir: str,
|
||||
if path and status != "hash_mismatch":
|
||||
return path, status
|
||||
|
||||
# Large files from GitHub release assets — tried when local file is
|
||||
# Large files from GitHub release assets -tried when local file is
|
||||
# missing OR has a hash mismatch (wrong variant on disk)
|
||||
name = file_entry.get("name", "")
|
||||
sha1 = file_entry.get("sha1")
|
||||
@@ -362,7 +362,7 @@ def _collect_emulator_extras(
|
||||
# Second pass: find alternative destinations for files already in the pack.
|
||||
# A file declared by the platform or emitted above may also be needed at a
|
||||
# different path by another core (e.g. neocd/ vs root, same_cdi/bios/ vs root).
|
||||
# Only adds a copy when the file is ALREADY covered at a different path —
|
||||
# Only adds a copy when the file is ALREADY covered at a different path -
|
||||
# never introduces a file that wasn't selected by the first pass.
|
||||
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir)
|
||||
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
|
||||
@@ -831,7 +831,7 @@ def generate_pack(
|
||||
|
||||
# Emulator-level validation: informational only for platform packs.
|
||||
# Platform verification (existence/md5) is the authority for pack status.
|
||||
# Emulator checks are supplementary — logged but don't downgrade.
|
||||
# Emulator checks are supplementary -logged but don't downgrade.
|
||||
# When a discrepancy is found, try to find a file satisfying both.
|
||||
if (file_status.get(dedup_key) == "ok"
|
||||
and local_path and validation_index):
|
||||
@@ -892,7 +892,7 @@ def generate_pack(
|
||||
if base_dest:
|
||||
full_dest = f"{base_dest}/{dest}"
|
||||
elif "/" not in dest:
|
||||
# Bare filename with empty base_destination — infer bios/ prefix
|
||||
# Bare filename with empty base_destination -infer bios/ prefix
|
||||
# to match platform conventions (RetroDECK: ~/retrodeck/bios/)
|
||||
full_dest = f"bios/{dest}"
|
||||
else:
|
||||
@@ -936,7 +936,7 @@ def generate_pack(
|
||||
continue
|
||||
local_path = entry.get("local_cache", "")
|
||||
if not local_path or not os.path.isdir(local_path):
|
||||
print(f" WARNING: data directory '{ref_key}' not cached at {local_path} — run refresh_data_dirs.py")
|
||||
print(f" WARNING: data directory '{ref_key}' not cached at {local_path} -run refresh_data_dirs.py")
|
||||
continue
|
||||
dd_dest = dd.get("destination", "")
|
||||
if base_dest and dd_dest:
|
||||
@@ -961,7 +961,7 @@ def generate_pack(
|
||||
zf.write(src, full)
|
||||
total_files += 1
|
||||
|
||||
# README.txt for users — personalized step-by-step per platform
|
||||
# README.txt for users -personalized step-by-step per platform
|
||||
num_systems = len(pack_systems)
|
||||
readme_text = _build_readme(platform_name, platform_display,
|
||||
base_dest, total_files, num_systems)
|
||||
@@ -983,7 +983,7 @@ def generate_pack(
|
||||
for key, reason in sorted(file_reasons.items()):
|
||||
status = file_status.get(key, "")
|
||||
label = "UNTESTED" if status == "untested" else "DISCREPANCY"
|
||||
print(f" {label}: {key} — {reason}")
|
||||
print(f" {label}: {key} -{reason}")
|
||||
for name in missing_files:
|
||||
print(f" MISSING: {name}")
|
||||
return zip_path
|
||||
@@ -1011,7 +1011,7 @@ def _normalize_zip_for_pack(source_zip: str, dest_path: str, target_zf: zipfile.
|
||||
the normalized version into the pack.
|
||||
|
||||
This ensures:
|
||||
- Same ROMs → same ZIP hash in every pack build
|
||||
- Same ROMs -> same ZIP hash in every pack build
|
||||
- No dependency on how the user built their MAME ROM set
|
||||
- Bit-identical ZIPs across platforms and build times
|
||||
"""
|
||||
@@ -1025,9 +1025,7 @@ def _normalize_zip_for_pack(source_zip: str, dest_path: str, target_zf: zipfile.
|
||||
os.unlink(tmp_path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emulator/system mode pack generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _resolve_destination(file_entry: dict, pack_structure: dict | None,
|
||||
standalone: bool) -> str:
|
||||
@@ -1081,11 +1079,11 @@ def generate_emulator_pack(
|
||||
p = all_profiles[name]
|
||||
if p.get("type") == "alias":
|
||||
alias_of = p.get("alias_of", "?")
|
||||
print(f"Error: {name} is an alias of {alias_of} — use --emulator {alias_of}",
|
||||
print(f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}",
|
||||
file=sys.stderr)
|
||||
return None
|
||||
if p.get("type") == "launcher":
|
||||
print(f"Error: {name} is a launcher — use the emulator it launches",
|
||||
print(f"Error: {name} is a launcher -use the emulator it launches",
|
||||
file=sys.stderr)
|
||||
return None
|
||||
ptype = p.get("type", "libretro")
|
||||
@@ -1931,9 +1929,7 @@ def main():
|
||||
emu_profiles, target_cores_cache, system_filter)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manifest generation (JSON inventory for install.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_GITIGNORE_ENTRIES: set[str] | None = None
|
||||
|
||||
@@ -2139,7 +2135,7 @@ def generate_manifest(
|
||||
if case_insensitive:
|
||||
seen_lower.add(full_dest.lower())
|
||||
|
||||
# No phase 3 (data directories) — skipped for manifest
|
||||
# No phase 3 (data directories) -skipped for manifest
|
||||
|
||||
now = __import__("datetime").datetime.now(
|
||||
__import__("datetime").timezone.utc
|
||||
@@ -2161,9 +2157,7 @@ def generate_manifest(
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Post-generation pack verification + manifest + SHA256SUMS
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def verify_pack(zip_path: str, db: dict,
|
||||
data_registry: dict | None = None) -> tuple[bool, dict]:
|
||||
@@ -2462,7 +2456,7 @@ def verify_pack_against_platform(
|
||||
|
||||
if full in zip_set or full.lower() in zip_lower:
|
||||
core_present += 1
|
||||
# Not an error if missing — some get deduped or filtered
|
||||
# Not an error if missing -some get deduped or filtered
|
||||
|
||||
checked = baseline_checked + core_checked
|
||||
present = baseline_present + core_present
|
||||
|
||||
@@ -189,9 +189,7 @@ def _status_icon(pct: float) -> str:
|
||||
return "partial"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Home page
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_home(db: dict, coverages: dict, profiles: dict,
|
||||
registry: dict | None = None) -> str:
|
||||
@@ -303,9 +301,7 @@ def generate_home(db: dict, coverages: dict, profiles: dict,
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Platform pages
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_platform_index(coverages: dict) -> str:
|
||||
lines = [
|
||||
@@ -478,9 +474,7 @@ def generate_platform_page(name: str, cov: dict, registry: dict | None = None,
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System pages
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]:
|
||||
"""Group files by manufacturer -> console -> files."""
|
||||
@@ -572,9 +566,7 @@ def generate_system_page(
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emulator pages
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_emulators_index(profiles: dict) -> str:
|
||||
unique = {k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test")}
|
||||
@@ -1011,9 +1003,7 @@ def generate_emulator_page(name: str, profile: dict, db: dict,
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Contributing page
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_gap_analysis(
|
||||
profiles: dict,
|
||||
@@ -1367,9 +1357,7 @@ The CI automatically:
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wiki pages
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_wiki_index() -> str:
|
||||
"""Generate wiki landing page."""
|
||||
@@ -1924,9 +1912,7 @@ def generate_wiki_data_model(db: dict, profiles: dict) -> str:
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Build cross-reference indexes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_platform_file_index(coverages: dict) -> dict[str, set]:
|
||||
"""Map platform_name -> set of declared file names."""
|
||||
@@ -1954,9 +1940,7 @@ def _build_emulator_file_index(profiles: dict) -> dict[str, dict]:
|
||||
return index
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# mkdocs.yml nav generator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def generate_mkdocs_nav(
|
||||
coverages: dict,
|
||||
@@ -2028,9 +2012,7 @@ def generate_mkdocs_nav(
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate MkDocs site from project data")
|
||||
@@ -2053,14 +2035,12 @@ def main():
|
||||
for d in GENERATED_DIRS:
|
||||
(docs / d).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load registry for platform metadata (logos, etc.)
|
||||
registry_path = Path(args.platforms_dir) / "_registry.yml"
|
||||
registry = {}
|
||||
if registry_path.exists():
|
||||
with open(registry_path) as f:
|
||||
registry = (yaml.safe_load(f) or {}).get("platforms", {})
|
||||
|
||||
# Load platform configs
|
||||
platform_names = list_registered_platforms(args.platforms_dir, include_archived=True)
|
||||
|
||||
print("Computing platform coverage...")
|
||||
@@ -2073,7 +2053,6 @@ def main():
|
||||
except FileNotFoundError as e:
|
||||
print(f" {name}: skipped ({e})", file=sys.stderr)
|
||||
|
||||
# Load emulator profiles
|
||||
print("Loading emulator profiles...")
|
||||
profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
|
||||
unique_count = sum(1 for p in profiles.values() if p.get("type") != "alias")
|
||||
|
||||
@@ -24,12 +24,12 @@ SOURCE_URL = (
|
||||
|
||||
# Libretro cores that expect BIOS files in a subdirectory of system/.
|
||||
# System.dat lists filenames flat; the scraper prepends the prefix.
|
||||
# ref: each core's libretro.c or equivalent — see platforms/README.md
|
||||
# ref: each core's libretro.c or equivalent -see platforms/README.md
|
||||
CORE_SUBDIR_MAP = {
|
||||
"nec-pc-98": "np2kai", # libretro-np2kai/sdl/libretro.c
|
||||
"sharp-x68000": "keropi", # px68k/libretro/libretro.c
|
||||
"sega-dreamcast": "dc", # flycast/shell/libretro/libretro.cpp
|
||||
"sega-dreamcast-arcade": "dc", # flycast — same subfolder
|
||||
"sega-dreamcast-arcade": "dc", # flycast -same subfolder
|
||||
}
|
||||
|
||||
SYSTEM_SLUG_MAP = {
|
||||
@@ -254,7 +254,7 @@ class Scraper(BaseScraper):
|
||||
|
||||
systems[req.system]["files"].append(entry)
|
||||
|
||||
# Systems not in System.dat but needed for RetroArch — added via
|
||||
# Systems not in System.dat but needed for RetroArch -added via
|
||||
# shared groups in _shared.yml. The includes directive is resolved
|
||||
# at load time by load_platform_config().
|
||||
EXTRA_SYSTEMS = {
|
||||
@@ -264,7 +264,7 @@ class Scraper(BaseScraper):
|
||||
"manufacturer": "NEC",
|
||||
"docs": "https://docs.libretro.com/library/quasi88/",
|
||||
},
|
||||
# ref: Vircon32/libretro.c — virtual console, single BIOS
|
||||
# ref: Vircon32/libretro.c -virtual console, single BIOS
|
||||
"vircon32": {
|
||||
"files": [
|
||||
{"name": "Vircon32Bios.v32", "destination": "Vircon32Bios.v32", "required": True},
|
||||
@@ -273,7 +273,7 @@ class Scraper(BaseScraper):
|
||||
"manufacturer": "Vircon",
|
||||
"docs": "https://docs.libretro.com/library/vircon32/",
|
||||
},
|
||||
# ref: xrick/src/sysvid.c, xrick/src/data.c — game data archive
|
||||
# ref: xrick/src/sysvid.c, xrick/src/data.c -game data archive
|
||||
"xrick": {
|
||||
"files": [
|
||||
{"name": "data.zip", "destination": "xrick/data.zip", "required": True},
|
||||
@@ -290,7 +290,7 @@ class Scraper(BaseScraper):
|
||||
# Arcade BIOS present in the repo but absent from System.dat.
|
||||
# FBNeo expects them in system/ or system/fbneo/.
|
||||
# ref: fbneo/src/burner/libretro/libretro.cpp
|
||||
# ref: fbneo/src/burner/libretro/libretro.cpp — search order:
|
||||
# ref: fbneo/src/burner/libretro/libretro.cpp -search order:
|
||||
# 1) romset dir 2) system/fbneo/ 3) system/
|
||||
EXTRA_ARCADE_FILES = [
|
||||
{"name": "namcoc69.zip", "destination": "namcoc69.zip", "required": True},
|
||||
@@ -329,33 +329,33 @@ class Scraper(BaseScraper):
|
||||
# Extra files missing from System.dat for specific systems.
|
||||
# Each traced to the core's source code.
|
||||
EXTRA_SYSTEM_FILES = {
|
||||
# melonDS DS DSi mode — ref: JesseTG/melonds-ds/src/libretro.cpp
|
||||
# melonDS DS DSi mode -ref: JesseTG/melonds-ds/src/libretro.cpp
|
||||
"nintendo-ds": [
|
||||
{"name": "dsi_bios7.bin", "destination": "dsi_bios7.bin", "required": True},
|
||||
{"name": "dsi_bios9.bin", "destination": "dsi_bios9.bin", "required": True},
|
||||
{"name": "dsi_firmware.bin", "destination": "dsi_firmware.bin", "required": True},
|
||||
{"name": "dsi_nand.bin", "destination": "dsi_nand.bin", "required": True},
|
||||
],
|
||||
# bsnes SGB naming — ref: bsnes/target-libretro/libretro.cpp
|
||||
# bsnes SGB naming -ref: bsnes/target-libretro/libretro.cpp
|
||||
"nintendo-sgb": [
|
||||
{"name": "sgb.boot.rom", "destination": "sgb.boot.rom", "required": False},
|
||||
],
|
||||
# JollyCV — ref: jollycv/libretro.c
|
||||
# JollyCV -ref: jollycv/libretro.c
|
||||
"coleco-colecovision": [
|
||||
{"name": "BIOS.col", "destination": "BIOS.col", "required": True},
|
||||
{"name": "coleco.rom", "destination": "coleco.rom", "required": True},
|
||||
{"name": "bioscv.rom", "destination": "bioscv.rom", "required": True},
|
||||
],
|
||||
# Kronos ST-V — ref: libretro-kronos/libretro/libretro.c
|
||||
# Kronos ST-V -ref: libretro-kronos/libretro/libretro.c
|
||||
"sega-saturn": [
|
||||
{"name": "stvbios.zip", "destination": "kronos/stvbios.zip", "required": True},
|
||||
],
|
||||
# PCSX ReARMed / Beetle PSX alt BIOS — ref: pcsx_rearmed/libpcsxcore/misc.c
|
||||
# PCSX ReARMed / Beetle PSX alt BIOS -ref: pcsx_rearmed/libpcsxcore/misc.c
|
||||
# docs say PSXONPSP660.bin (uppercase) but core accepts any case
|
||||
"sony-playstation": [
|
||||
{"name": "psxonpsp660.bin", "destination": "psxonpsp660.bin", "required": False},
|
||||
],
|
||||
# Dolphin GC — ref: DolphinLibretro/Boot.cpp:72-73,
|
||||
# Dolphin GC -ref: DolphinLibretro/Boot.cpp:72-73,
|
||||
# BootManager.cpp:200-217, CommonPaths.h:139 GC_IPL="IPL.bin"
|
||||
# Core searches system/dolphin-emu/Sys/ for data and BIOS.
|
||||
# System.dat gc-ntsc-*.bin names are NOT what Dolphin loads.
|
||||
@@ -364,15 +364,15 @@ class Scraper(BaseScraper):
|
||||
{"name": "gc-ntsc-12.bin", "destination": "dolphin-emu/Sys/GC/USA/IPL.bin", "required": False},
|
||||
{"name": "gc-pal-12.bin", "destination": "dolphin-emu/Sys/GC/EUR/IPL.bin", "required": False},
|
||||
{"name": "gc-ntsc-12.bin", "destination": "dolphin-emu/Sys/GC/JAP/IPL.bin", "required": False},
|
||||
# DSP firmware — ref: Source/Core/Core/HW/DSPLLE/DSPHost.cpp
|
||||
# DSP firmware -ref: Source/Core/Core/HW/DSPLLE/DSPHost.cpp
|
||||
{"name": "dsp_coef.bin", "destination": "dolphin-emu/Sys/GC/dsp_coef.bin", "required": True},
|
||||
{"name": "dsp_rom.bin", "destination": "dolphin-emu/Sys/GC/dsp_rom.bin", "required": True},
|
||||
# Fonts — ref: Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp
|
||||
# Fonts -ref: Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp
|
||||
{"name": "font_western.bin", "destination": "dolphin-emu/Sys/GC/font_western.bin", "required": False},
|
||||
{"name": "font_japanese.bin", "destination": "dolphin-emu/Sys/GC/font_japanese.bin", "required": False},
|
||||
],
|
||||
# minivmac casing — ref: minivmac/src/MYOSGLUE.c
|
||||
# doc says MacII.rom, repo has MacII.ROM — both work on case-insensitive FS
|
||||
# minivmac casing -ref: minivmac/src/MYOSGLUE.c
|
||||
# doc says MacII.rom, repo has MacII.ROM -both work on case-insensitive FS
|
||||
"apple-macintosh-ii": [
|
||||
{"name": "MacII.ROM", "destination": "MacII.ROM", "required": True},
|
||||
],
|
||||
@@ -398,7 +398,7 @@ class Scraper(BaseScraper):
|
||||
# Inject shared group references for systems that have core-specific
|
||||
# subdirectory requirements already defined in _shared.yml.
|
||||
# Note: fuse/ prefix NOT injected for sinclair-zx-spectrum.
|
||||
# Verified in fuse-libretro/src/compat/paths.c — core searches
|
||||
# Verified in fuse-libretro/src/compat/paths.c -core searches
|
||||
# system/ flat, not fuse/ subfolder. Docs are wrong on this.
|
||||
SYSTEM_SHARED_GROUPS = {
|
||||
"nec-pc-98": ["np2kai"],
|
||||
@@ -421,12 +421,12 @@ class Scraper(BaseScraper):
|
||||
{"ref": "ppsspp-assets", "destination": "PPSSPP"},
|
||||
],
|
||||
# single buildbot ZIP contains both Databases/ and Machines/
|
||||
# ref: libretro.c:1118-1119 — system_dir/Machines + system_dir/Databases
|
||||
# ref: libretro.c:1118-1119 -system_dir/Machines + system_dir/Databases
|
||||
"microsoft-msx": [
|
||||
{"ref": "bluemsx", "destination": ""},
|
||||
],
|
||||
# FreeIntv overlays — system/freeintv_overlays/<rom>.png
|
||||
# ref: FreeIntv/src/libretro.c:273 — stbi_load from system dir
|
||||
# FreeIntv overlays -system/freeintv_overlays/<rom>.png
|
||||
# ref: FreeIntv/src/libretro.c:273 -stbi_load from system dir
|
||||
# ZIP contains FreeIntvTS_Overlays/ subfolder, cache preserves it
|
||||
# pack destination maps cache root to system/freeintv_overlays
|
||||
# so final path is system/freeintv_overlays/FreeIntvTS_Overlays/<rom>.png
|
||||
|
||||
@@ -171,7 +171,7 @@ def _resolve_path(p: str) -> str:
|
||||
def _extract_bios_entries(component_val: dict) -> list[dict]:
|
||||
"""Extract BIOS entries from all three possible locations in a component.
|
||||
|
||||
No dedup here — dedup is done in fetch_requirements() with full
|
||||
No dedup here -dedup is done in fetch_requirements() with full
|
||||
(system, filename) key to avoid dropping valid same-filename entries
|
||||
across different systems.
|
||||
"""
|
||||
@@ -338,13 +338,13 @@ class Scraper(BaseScraper):
|
||||
if resolved.startswith("saves"):
|
||||
continue
|
||||
|
||||
# Build destination — default to bios/ if no path specified
|
||||
# Build destination -default to bios/ if no path specified
|
||||
if resolved:
|
||||
destination = f"{resolved}/{filename}"
|
||||
else:
|
||||
destination = f"bios/{filename}"
|
||||
|
||||
# MD5 handling — sanitize upstream errors
|
||||
# MD5 handling -sanitize upstream errors
|
||||
md5_raw = entry.get("md5", "")
|
||||
if isinstance(md5_raw, list):
|
||||
parts = [str(m).strip().lower() for m in md5_raw if m]
|
||||
|
||||
@@ -136,7 +136,7 @@ def _check_atom(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bo
|
||||
if tok.startswith('"'):
|
||||
pos += 1
|
||||
return True, pos
|
||||
# Unknown token — treat as true to avoid false negatives
|
||||
# Unknown token -treat as true to avoid false negatives
|
||||
pos += 1
|
||||
return True, pos
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""Scraper for EmuDeck emulator targets.
|
||||
|
||||
Sources:
|
||||
SteamOS: dragoonDorise/EmuDeck — functions/EmuScripts/*.sh
|
||||
Windows: EmuDeck/emudeck-we — functions/EmuScripts/*.ps1
|
||||
SteamOS: dragoonDorise/EmuDeck -functions/EmuScripts/*.sh
|
||||
Windows: EmuDeck/emudeck-we -functions/EmuScripts/*.ps1
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ TARGETS: list[tuple[str, str, str]] = [
|
||||
("nintendo/wiiu/latest", "nintendo-wiiu", "ppc"),
|
||||
("playstation/ps2/latest", "playstation-ps2", "mips"),
|
||||
("playstation/psp/latest", "playstation-psp", "mips"),
|
||||
# vita: only VPK bundles on buildbot — cores listed via libretro-super recipes
|
||||
# vita: only VPK bundles on buildbot -cores listed via libretro-super recipes
|
||||
]
|
||||
|
||||
# Recipe-based targets: (recipe_path_under_RECIPE_BASE_URL, target_name, architecture)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Implements GF(2^233) field arithmetic, elliptic curve point operations,
|
||||
and ECDSA-SHA256 verification for Nintendo 3DS OTP certificate checking.
|
||||
|
||||
Zero external dependencies — uses only Python stdlib.
|
||||
Zero external dependencies -uses only Python stdlib.
|
||||
|
||||
Curve: sect233r1 (NIST B-233, SEC 2 v2)
|
||||
Field: GF(2^233) with irreducible polynomial t^233 + t^74 + 1
|
||||
@@ -13,9 +13,7 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sect233r1 curve parameters (SEC 2 v2)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_M = 233
|
||||
_F = (1 << 233) | (1 << 74) | 1 # irreducible polynomial
|
||||
@@ -34,9 +32,7 @@ _N_BITLEN = _N.bit_length() # 233
|
||||
_H = 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GF(2^233) field arithmetic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _gf_reduce(a: int) -> int:
|
||||
"""Reduce polynomial a modulo t^233 + t^74 + 1."""
|
||||
@@ -85,7 +81,7 @@ def _gf_inv(a: int) -> int:
|
||||
q ^= 1 << shift
|
||||
temp ^= r << shift
|
||||
remainder = temp
|
||||
# Multiply q * s in GF(2)[x] (no reduction — working in polynomial ring)
|
||||
# Multiply q * s in GF(2)[x] (no reduction -working in polynomial ring)
|
||||
qs = 0
|
||||
qt = q
|
||||
st = s
|
||||
@@ -102,10 +98,8 @@ def _gf_inv(a: int) -> int:
|
||||
return _gf_reduce(old_s)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Elliptic curve point operations on sect233r1
|
||||
# y^2 + xy = x^3 + ax^2 + b (a=1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Point at infinity
|
||||
_INF = None
|
||||
@@ -175,9 +169,7 @@ def _ec_mul(k: int, p: tuple[int, int] | None) -> tuple[int, int] | None:
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ECDSA-SHA256 verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _modinv(a: int, m: int) -> int:
|
||||
"""Modular inverse of a modulo m (integers, not GF(2^m))."""
|
||||
|
||||
@@ -217,7 +217,7 @@ def generate_platform_truth(
|
||||
sys_cov["profiled"].add(emu_name)
|
||||
|
||||
# Ensure all systems of resolved cores have entries (even with 0 files).
|
||||
# This documents that the system is covered — the core was analyzed and
|
||||
# This documents that the system is covered -the core was analyzed and
|
||||
# needs no external files for this system.
|
||||
for emu_name in cores_profiled:
|
||||
profile = profiles[emu_name]
|
||||
@@ -261,9 +261,7 @@ def generate_platform_truth(
|
||||
}
|
||||
|
||||
|
||||
# -------------------------------------------------------------------
|
||||
# Platform truth diffing
|
||||
# -------------------------------------------------------------------
|
||||
|
||||
def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
|
||||
"""Compare files between truth and scraped for a single system."""
|
||||
@@ -430,7 +428,7 @@ def diff_platform_truth(truth: dict, scraped: dict) -> dict:
|
||||
else:
|
||||
summary["systems_fully_covered"] += 1
|
||||
|
||||
# Truth systems not matched by any scraped system — all files missing
|
||||
# Truth systems not matched by any scraped system -all files missing
|
||||
for t_sid in sorted(truth_systems):
|
||||
if t_sid in matched_truth:
|
||||
continue
|
||||
|
||||
@@ -12,7 +12,7 @@ import os
|
||||
from common import compute_hashes
|
||||
|
||||
# Validation types that require console-specific cryptographic keys.
|
||||
# verify.py cannot reproduce these — size checks still apply if combined.
|
||||
# verify.py cannot reproduce these -size checks still apply if combined.
|
||||
_CRYPTO_CHECKS = frozenset({"signature", "crypto"})
|
||||
|
||||
# All reproducible validation types.
|
||||
@@ -85,7 +85,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
|
||||
if f.get("max_size") is not None:
|
||||
cur = index[fname]["max_size"]
|
||||
index[fname]["max_size"] = max(cur, f["max_size"]) if cur is not None else f["max_size"]
|
||||
# Hash checks — collect all accepted hashes as sets (multiple valid
|
||||
# Hash checks -collect all accepted hashes as sets (multiple valid
|
||||
# versions of the same file, e.g. MT-32 ROM versions)
|
||||
if "crc32" in checks and f.get("crc32"):
|
||||
crc_val = f["crc32"]
|
||||
@@ -103,7 +103,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
|
||||
index[fname][hash_type].add(str(h).lower())
|
||||
else:
|
||||
index[fname][hash_type].add(str(val).lower())
|
||||
# Adler32 — stored as known_hash_adler32 field (not in validation: list
|
||||
# Adler32 -stored as known_hash_adler32 field (not in validation: list
|
||||
# for Dolphin, but support it in both forms for future profiles)
|
||||
adler_val = f.get("known_hash_adler32") or f.get("adler32")
|
||||
if adler_val:
|
||||
@@ -186,7 +186,7 @@ def check_file_validation(
|
||||
return None
|
||||
checks = entry["checks"]
|
||||
|
||||
# Size checks — sizes is a set of accepted values
|
||||
# Size checks -sizes is a set of accepted values
|
||||
if "size" in checks:
|
||||
actual_size = os.path.getsize(local_path)
|
||||
if entry["sizes"] and actual_size not in entry["sizes"]:
|
||||
@@ -197,7 +197,7 @@ def check_file_validation(
|
||||
if entry["max_size"] is not None and actual_size > entry["max_size"]:
|
||||
return f"size too large: max {entry['max_size']}, got {actual_size}"
|
||||
|
||||
# Hash checks — compute once, reuse for all hash types.
|
||||
# Hash checks -compute once, reuse for all hash types.
|
||||
# Each hash field is a set of accepted values (multiple valid ROM versions).
|
||||
need_hashes = (
|
||||
any(h in checks and entry.get(h) for h in ("crc32", "md5", "sha1", "sha256"))
|
||||
|
||||
@@ -47,9 +47,7 @@ DEFAULT_PLATFORMS_DIR = "platforms"
|
||||
DEFAULT_EMULATORS_DIR = "emulators"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Status model — aligned with Batocera BiosStatus (batocera-systems:967-969)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Status model -aligned with Batocera BiosStatus (batocera-systems:967-969)
|
||||
|
||||
class Status:
|
||||
OK = "ok"
|
||||
@@ -68,15 +66,13 @@ _STATUS_ORDER = {Status.OK: 0, Status.UNTESTED: 1, Status.MISSING: 2}
|
||||
_SEVERITY_ORDER = {Severity.OK: 0, Severity.INFO: 1, Severity.WARNING: 2, Severity.CRITICAL: 3}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Verification functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def verify_entry_existence(
|
||||
file_entry: dict, local_path: str | None,
|
||||
validation_index: dict[str, dict] | None = None,
|
||||
) -> dict:
|
||||
"""RetroArch verification: path_is_valid() — file exists = OK."""
|
||||
"""RetroArch verification: path_is_valid() -file exists = OK."""
|
||||
name = file_entry.get("name", "")
|
||||
required = file_entry.get("required", True)
|
||||
if not local_path:
|
||||
@@ -96,7 +92,7 @@ def verify_entry_md5(
|
||||
local_path: str | None,
|
||||
resolve_status: str = "",
|
||||
) -> dict:
|
||||
"""MD5 verification — Batocera md5sum + Recalbox multi-hash + Md5Composite."""
|
||||
"""MD5 verification -Batocera md5sum + Recalbox multi-hash + Md5Composite."""
|
||||
name = file_entry.get("name", "")
|
||||
expected_md5 = file_entry.get("md5", "")
|
||||
zipped_file = file_entry.get("zipped_file")
|
||||
@@ -162,7 +158,7 @@ def verify_entry_sha1(
|
||||
file_entry: dict,
|
||||
local_path: str | None,
|
||||
) -> dict:
|
||||
"""SHA1 verification — BizHawk firmware hash check."""
|
||||
"""SHA1 verification -BizHawk firmware hash check."""
|
||||
name = file_entry.get("name", "")
|
||||
expected_sha1 = file_entry.get("sha1", "")
|
||||
required = file_entry.get("required", True)
|
||||
@@ -183,20 +179,18 @@ def verify_entry_sha1(
|
||||
"reason": f"expected {expected_sha1[:12]}… got {actual_sha1[:12]}…"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Severity mapping per platform
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def compute_severity(
|
||||
status: str, required: bool, mode: str, hle_fallback: bool = False,
|
||||
) -> str:
|
||||
"""Map (status, required, verification_mode, hle_fallback) → severity.
|
||||
"""Map (status, required, verification_mode, hle_fallback) -> severity.
|
||||
|
||||
Based on native platform behavior + emulator HLE capability:
|
||||
- RetroArch (existence): required+missing = warning, optional+missing = info
|
||||
- Batocera/Recalbox/RetroBat/EmuDeck (md5): hash-based verification
|
||||
- BizHawk (sha1): same severity rules as md5
|
||||
- hle_fallback: core works without this file via HLE → always INFO when missing
|
||||
- hle_fallback: core works without this file via HLE -> always INFO when missing
|
||||
"""
|
||||
if status == Status.OK:
|
||||
return Severity.OK
|
||||
@@ -218,13 +212,9 @@ def compute_severity(
|
||||
return Severity.OK
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ZIP content index
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-reference: undeclared files used by cores
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_expected(file_entry: dict, checks: list[str]) -> dict:
|
||||
@@ -447,7 +437,7 @@ def find_exclusion_notes(
|
||||
})
|
||||
continue
|
||||
|
||||
# Count standalone-only files — but only report as excluded if the
|
||||
# Count standalone-only files -but only report as excluded if the
|
||||
# platform does NOT use this emulator in standalone mode
|
||||
standalone_set = set(str(c) for c in config.get("standalone_cores", []))
|
||||
is_standalone = emu_name in standalone_set or bool(
|
||||
@@ -466,9 +456,7 @@ def find_exclusion_notes(
|
||||
return notes
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Platform verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _find_best_variant(
|
||||
file_entry: dict, db: dict, current_path: str,
|
||||
@@ -640,9 +628,7 @@ def verify_platform(
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _format_ground_truth_aggregate(ground_truth: list[dict]) -> str:
|
||||
"""Format ground truth as a single aggregated line.
|
||||
@@ -698,7 +684,7 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) ->
|
||||
req = "required" if d.get("required", True) else "optional"
|
||||
hle = ", HLE available" if d.get("hle_fallback") else ""
|
||||
reason = d.get("reason", "")
|
||||
print(f" UNTESTED ({req}{hle}): {key} — {reason}")
|
||||
print(f" UNTESTED ({req}{hle}): {key} -{reason}")
|
||||
_print_ground_truth(d.get("ground_truth", []), verbose)
|
||||
for d in details:
|
||||
if d["status"] == Status.MISSING:
|
||||
@@ -717,7 +703,7 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) ->
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
print(f" DISCREPANCY: {key} — {disc}")
|
||||
print(f" DISCREPANCY: {key} -{disc}")
|
||||
_print_ground_truth(d.get("ground_truth", []), verbose)
|
||||
|
||||
if verbose:
|
||||
@@ -831,7 +817,7 @@ def print_platform_result(result: dict, group: list[str], verbose: bool = False)
|
||||
if exclusions:
|
||||
print(f" No external files ({len(exclusions)}):")
|
||||
for ex in exclusions:
|
||||
print(f" {ex['emulator']} — {ex['detail']} [{ex['reason']}]")
|
||||
print(f" {ex['emulator']} -{ex['detail']} [{ex['reason']}]")
|
||||
|
||||
gt_cov = result.get("ground_truth_coverage")
|
||||
if gt_cov and gt_cov["total"] > 0:
|
||||
@@ -841,9 +827,7 @@ def print_platform_result(result: dict, group: list[str], verbose: bool = False)
|
||||
print(f" {gt_cov['platform_only']} platform-only (no emulator profile)")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Emulator/system mode verification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _effective_validation_label(details: list[dict], validation_index: dict) -> str:
|
||||
"""Determine the bracket label for the report.
|
||||
@@ -892,11 +876,11 @@ def verify_emulator(
|
||||
p = all_profiles[name]
|
||||
if p.get("type") == "alias":
|
||||
alias_of = p.get("alias_of", "?")
|
||||
print(f"Error: {name} is an alias of {alias_of} — use --emulator {alias_of}",
|
||||
print(f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if p.get("type") == "launcher":
|
||||
print(f"Error: {name} is a launcher — use the emulator it launches",
|
||||
print(f"Error: {name} is a launcher -use the emulator it launches",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
# Check standalone capability
|
||||
@@ -1117,7 +1101,7 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
|
||||
req = "required" if d.get("required", True) else "optional"
|
||||
hle = ", HLE available" if d.get("hle_fallback") else ""
|
||||
reason = d.get("reason", "")
|
||||
print(f" UNTESTED ({req}{hle}): {d['name']} — {reason}")
|
||||
print(f" UNTESTED ({req}{hle}): {d['name']} -{reason}")
|
||||
gt = d.get("ground_truth", [])
|
||||
if gt:
|
||||
if verbose:
|
||||
|
||||
@@ -230,10 +230,10 @@ class TestE2E(unittest.TestCase):
|
||||
# Correct hash
|
||||
{"name": "correct_hash.bin", "destination": "correct_hash.bin",
|
||||
"md5": f["correct_hash.bin"]["md5"], "required": True},
|
||||
# Wrong hash on disk → untested
|
||||
# Wrong hash on disk ->untested
|
||||
{"name": "wrong_hash.bin", "destination": "wrong_hash.bin",
|
||||
"md5": "ffffffffffffffffffffffffffffffff", "required": True},
|
||||
# No MD5 → OK (existence within md5 platform)
|
||||
# No MD5 ->OK (existence within md5 platform)
|
||||
{"name": "no_md5.bin", "destination": "no_md5.bin", "required": False},
|
||||
# Missing required
|
||||
{"name": "gone_req.bin", "destination": "gone_req.bin",
|
||||
@@ -259,7 +259,7 @@ class TestE2E(unittest.TestCase):
|
||||
# Truncated MD5 (Batocera 29 chars)
|
||||
{"name": "truncated.bin", "destination": "truncated.bin",
|
||||
"md5": truncated_md5, "required": True},
|
||||
# Same destination from different entry → worst status wins
|
||||
# Same destination from different entry ->worst status wins
|
||||
{"name": "correct_hash.bin", "destination": "dedup_target.bin",
|
||||
"md5": f["correct_hash.bin"]["md5"], "required": True},
|
||||
{"name": "correct_hash.bin", "destination": "dedup_target.bin",
|
||||
@@ -367,7 +367,7 @@ class TestE2E(unittest.TestCase):
|
||||
with open(os.path.join(self.emulators_dir, "test_alias.yml"), "w") as fh:
|
||||
yaml.dump(alias, fh)
|
||||
|
||||
# Emulator with data_dir that matches platform → gaps suppressed
|
||||
# Emulator with data_dir that matches platform ->gaps suppressed
|
||||
emu_dd = {
|
||||
"emulator": "TestEmuDD",
|
||||
"type": "libretro",
|
||||
@@ -415,39 +415,39 @@ class TestE2E(unittest.TestCase):
|
||||
"type": "libretro",
|
||||
"systems": ["console-a", "sys-md5"],
|
||||
"files": [
|
||||
# Size validation — correct size (16 bytes = len(b"PRESENT_REQUIRED"))
|
||||
# Size validation -correct size (16 bytes = len(b"PRESENT_REQUIRED"))
|
||||
{"name": "present_req.bin", "required": True,
|
||||
"validation": ["size"], "size": 16,
|
||||
"source_ref": "test.c:10-20"},
|
||||
# Size validation — wrong expected size
|
||||
# Size validation -wrong expected size
|
||||
{"name": "present_opt.bin", "required": False,
|
||||
"validation": ["size"], "size": 9999},
|
||||
# CRC32 validation — correct crc32
|
||||
# CRC32 validation -correct crc32
|
||||
{"name": "correct_hash.bin", "required": True,
|
||||
"validation": ["crc32"], "crc32": "91d0b1d3",
|
||||
"source_ref": "hash.c:42"},
|
||||
# CRC32 validation — wrong crc32
|
||||
# CRC32 validation -wrong crc32
|
||||
{"name": "no_md5.bin", "required": False,
|
||||
"validation": ["crc32"], "crc32": "deadbeef"},
|
||||
# CRC32 starting with '0' (regression: lstrip("0x") bug)
|
||||
{"name": "leading_zero_crc.bin", "required": True,
|
||||
"validation": ["crc32"], "crc32": "0179e92e"},
|
||||
# MD5 validation — correct md5
|
||||
# MD5 validation -correct md5
|
||||
{"name": "correct_hash.bin", "required": True,
|
||||
"validation": ["md5"], "md5": "4a8db431e3b1a1acacec60e3424c4ce8"},
|
||||
# SHA1 validation — correct sha1
|
||||
# SHA1 validation -correct sha1
|
||||
{"name": "correct_hash.bin", "required": True,
|
||||
"validation": ["sha1"], "sha1": "a2ab6c95c5bbd191b9e87e8f4e85205a47be5764"},
|
||||
# MD5 validation — wrong md5
|
||||
# MD5 validation -wrong md5
|
||||
{"name": "alias_target.bin", "required": False,
|
||||
"validation": ["md5"], "md5": "0000000000000000000000000000dead"},
|
||||
# Adler32 — known_hash_adler32 field
|
||||
# Adler32 -known_hash_adler32 field
|
||||
{"name": "present_req.bin", "required": True,
|
||||
"known_hash_adler32": None}, # placeholder, set below
|
||||
# Min/max size range validation
|
||||
{"name": "present_req.bin", "required": True,
|
||||
"validation": ["size"], "min_size": 10, "max_size": 100},
|
||||
# Signature — crypto check we can't reproduce, but size applies
|
||||
# Signature -crypto check we can't reproduce, but size applies
|
||||
{"name": "correct_hash.bin", "required": True,
|
||||
"validation": ["size", "signature"], "size": 17},
|
||||
],
|
||||
@@ -491,7 +491,7 @@ class TestE2E(unittest.TestCase):
|
||||
yaml.dump(emu_subdir, fh)
|
||||
|
||||
# ---------------------------------------------------------------
|
||||
# THE TEST — one method per feature area, all using same fixtures
|
||||
# THE TEST -one method per feature area, all using same fixtures
|
||||
# ---------------------------------------------------------------
|
||||
|
||||
def test_01_resolve_sha1(self):
|
||||
@@ -676,7 +676,7 @@ class TestE2E(unittest.TestCase):
|
||||
profiles = load_emulator_profiles(self.emulators_dir)
|
||||
undeclared = find_undeclared_files(config, self.emulators_dir, self.db, profiles)
|
||||
names = {u["name"] for u in undeclared}
|
||||
# dd_covered.bin is a file entry, not data_dir content — still undeclared
|
||||
# dd_covered.bin is a file entry, not data_dir content -still undeclared
|
||||
self.assertIn("dd_covered.bin", names)
|
||||
|
||||
def test_44_cross_ref_skips_launchers(self):
|
||||
@@ -688,7 +688,7 @@ class TestE2E(unittest.TestCase):
|
||||
self.assertNotIn("launcher_bios.bin", names)
|
||||
|
||||
def test_45_hle_fallback_downgrades_severity(self):
|
||||
"""Missing file with hle_fallback=true → INFO severity, not CRITICAL."""
|
||||
"""Missing file with hle_fallback=true ->INFO severity, not CRITICAL."""
|
||||
from verify import compute_severity, Severity
|
||||
# required + missing + NO HLE = CRITICAL
|
||||
sev = compute_severity("missing", True, "md5", hle_fallback=False)
|
||||
@@ -723,7 +723,7 @@ class TestE2E(unittest.TestCase):
|
||||
groups = group_identical_platforms(
|
||||
["test_existence", "test_inherited"], self.platforms_dir
|
||||
)
|
||||
# Different base_destination → separate groups
|
||||
# Different base_destination ->separate groups
|
||||
self.assertEqual(len(groups), 2)
|
||||
|
||||
def test_51_platform_grouping_same(self):
|
||||
@@ -1064,7 +1064,7 @@ class TestE2E(unittest.TestCase):
|
||||
def test_95_verify_emulator_validation_applied(self):
|
||||
"""Emulator mode applies validation checks as primary verification."""
|
||||
result = verify_emulator(["test_validation"], self.emulators_dir, self.db)
|
||||
# present_opt.bin has wrong size → UNTESTED
|
||||
# present_opt.bin has wrong size ->UNTESTED
|
||||
for d in result["details"]:
|
||||
if d["name"] == "present_opt.bin":
|
||||
self.assertEqual(d["status"], Status.UNTESTED)
|
||||
@@ -1092,13 +1092,13 @@ class TestE2E(unittest.TestCase):
|
||||
def test_98_verify_emulator_validation_label(self):
|
||||
"""Validation label reflects the checks used."""
|
||||
result = verify_emulator(["test_validation"], self.emulators_dir, self.db)
|
||||
# test_validation has crc32, md5, sha1, size → all listed
|
||||
# test_validation has crc32, md5, sha1, size ->all listed
|
||||
self.assertEqual(result["verification_mode"], "crc32+md5+sha1+signature+size")
|
||||
|
||||
def test_99filter_files_by_mode(self):
|
||||
"""filter_files_by_mode correctly filters standalone/libretro."""
|
||||
files = [
|
||||
{"name": "a.bin"}, # no mode → both
|
||||
{"name": "a.bin"}, # no mode ->both
|
||||
{"name": "b.bin", "mode": "libretro"}, # libretro only
|
||||
{"name": "c.bin", "mode": "standalone"}, # standalone only
|
||||
{"name": "d.bin", "mode": "both"}, # explicit both
|
||||
@@ -1126,7 +1126,7 @@ class TestE2E(unittest.TestCase):
|
||||
self.assertTrue(len(notes) > 0)
|
||||
|
||||
def test_101_verify_emulator_severity_missing_required(self):
|
||||
"""Missing required file in emulator mode → WARNING severity."""
|
||||
"""Missing required file in emulator mode ->WARNING severity."""
|
||||
result = verify_emulator(["test_emu"], self.emulators_dir, self.db)
|
||||
# undeclared_req.bin is required and missing
|
||||
for d in result["details"]:
|
||||
@@ -1642,7 +1642,7 @@ class TestE2E(unittest.TestCase):
|
||||
config = load_platform_config("test_existence", self.platforms_dir)
|
||||
profiles = load_emulator_profiles(self.emulators_dir)
|
||||
result = verify_platform(config, self.db, self.emulators_dir, profiles)
|
||||
# Simulate --json filtering (non-OK only) — ground_truth must survive
|
||||
# Simulate --json filtering (non-OK only) -ground_truth must survive
|
||||
filtered = [d for d in result["details"] if d["status"] != Status.OK]
|
||||
for d in filtered:
|
||||
self.assertIn("ground_truth", d)
|
||||
@@ -2426,7 +2426,7 @@ class TestE2E(unittest.TestCase):
|
||||
config = load_platform_config("test_archive_platform", self.platforms_dir)
|
||||
profiles = load_emulator_profiles(self.emulators_dir)
|
||||
undeclared = find_undeclared_files(config, self.emulators_dir, self.db, profiles)
|
||||
# test_archive.zip is declared → its archived ROMs should be skipped
|
||||
# test_archive.zip is declared ->its archived ROMs should be skipped
|
||||
archive_entries = [u for u in undeclared if u.get("archive") == "test_archive.zip"]
|
||||
self.assertEqual(len(archive_entries), 0)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user