refactor: harden codebase and remove unicode artifacts

- fix urllib.parse.quote import (was urllib.request.quote)
- add operator precedence parens in generate_pack dedup check
- narrow bare except to specific types in batocera target scraper
- cache load_platform_config and build_zip_contents_index results
- add selective algorithm support to compute_hashes
- atomic write for fetch_large_file (tmp + rename)
- add response size limit to base scraper fetch
- extract build_target_cores_cache to common.py (dedup verify/pack)
- hoist _build_supplemental_index out of per-platform loop
- migrate function-attribute caches to module-level dicts
- add @abstractmethod to BaseTargetScraper.fetch_targets
- remove backward-compat re-exports from common.py
- replace em-dashes and unicode arrows with ASCII equivalents
- remove decorative section dividers and obvious comments
This commit is contained in:
Abdessamad Derraz
2026-03-29 23:15:20 +02:00
parent 0c5cde83e1
commit 2e21d64a08
17 changed files with 102 additions and 165 deletions

View File

@@ -471,7 +471,7 @@ def resolve_local_file(
if valid:
primary = [p for p, _ in valid if "/.variants/" not in p]
return (primary[0] if primary else valid[0][0]), "hash_mismatch"
# No candidate contains the zipped_file fall through to step 5
# No candidate contains the zipped_file -fall through to step 5
else:
primary = [p for p, _ in candidates if "/.variants/" not in p]
return (primary[0] if primary else candidates[0][0]), "hash_mismatch"
@@ -550,7 +550,7 @@ def _get_mame_clone_map() -> dict[str, str]:
def check_inside_zip(container: str, file_name: str, expected_md5: str) -> str:
"""Check a ROM inside a ZIP replicates Batocera checkInsideZip().
"""Check a ROM inside a ZIP -replicates Batocera checkInsideZip().
Returns "ok", "untested", "not_in_zip", or "error".
"""
@@ -765,7 +765,7 @@ MANUFACTURER_PREFIXES = (
"snk-", "panasonic-", "nec-", "epoch-", "mattel-", "fairchild-",
"hartung-", "tiger-", "magnavox-", "philips-", "bandai-", "casio-",
"coleco-", "commodore-", "sharp-", "sinclair-", "atari-", "sammy-",
"gce-", "texas-instruments-",
"gce-", "interton-", "texas-instruments-",
)
@@ -869,20 +869,20 @@ def filter_systems_by_target(
plat_cores_here = norm_plat_system_cores.get(norm_key, set())
if not all_cores and not plat_cores_here:
# No profile maps to this system keep it
# No profile maps to this system -keep it
filtered[sys_id] = sys_data
elif all_cores & expanded_target:
# At least one core is on the target
filtered[sys_id] = sys_data
elif not plat_cores_here:
# Platform resolution didn't find cores for this system keep it
# Platform resolution didn't find cores for this system -keep it
filtered[sys_id] = sys_data
# else: known cores exist but none are on the target exclude
# else: known cores exist but none are on the target -exclude
return filtered
# Validation and mode filtering extracted to validation.py for SoC.
# Validation and mode filtering -extracted to validation.py for SoC.
# Re-exported below for backward compatibility.

View File

@@ -23,9 +23,7 @@ from collections.abc import Callable
from pathlib import Path
# ---------------------------------------------------------------------------
# Key file parsing (keys.txt / aes_keys.txt format)
# ---------------------------------------------------------------------------
def parse_keys_file(path: str | Path) -> dict[str, dict[str, bytes]]:
"""Parse a 3DS keys file with :AES, :RSA, :ECC sections.
@@ -67,9 +65,7 @@ def find_keys_file(bios_dir: str | Path) -> Path | None:
return None
# ---------------------------------------------------------------------------
# Pure Python RSA-2048 PKCS1v15 SHA256 verification (zero dependencies)
# ---------------------------------------------------------------------------
def _rsa_verify_pkcs1v15_sha256(
message: bytes,
@@ -79,7 +75,7 @@ def _rsa_verify_pkcs1v15_sha256(
) -> bool:
"""Verify RSA-2048 PKCS#1 v1.5 with SHA-256.
Pure Python uses Python's native int for modular exponentiation.
Pure Python -uses Python's native int for modular exponentiation.
Reproduces CryptoPP::RSASS<PKCS1v15, SHA256>::Verifier.
"""
n = int.from_bytes(modulus, "big")
@@ -124,9 +120,7 @@ def _rsa_verify_pkcs1v15_sha256(
return em == expected_em
# ---------------------------------------------------------------------------
# AES-128-CBC decryption (with fallback)
# ---------------------------------------------------------------------------
def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
"""Decrypt AES-128-CBC without padding."""
@@ -166,9 +160,7 @@ def _aes_128_cbc_decrypt(data: bytes, key: bytes, iv: bytes) -> bytes:
)
# ---------------------------------------------------------------------------
# File verification functions
# ---------------------------------------------------------------------------
def verify_secure_info_a(
filepath: str | Path,
@@ -347,7 +339,7 @@ def verify_otp(
if computed_hash != stored_hash:
return False, "SHA-256 hash mismatch (OTP corrupted)"
# --- ECC certificate verification (sect233r1) ---
# ECC certificate verification (sect233r1)
ecc_keys = keys.get("ECC", {})
root_public_xy = ecc_keys.get("rootPublicXY")
if not root_public_xy or len(root_public_xy) != 60:
@@ -414,9 +406,7 @@ def verify_otp(
return False, "decrypted, magic+SHA256 valid, but ECC cert signature invalid"
# ---------------------------------------------------------------------------
# Unified verification interface for verify.py
# ---------------------------------------------------------------------------
# Map from (filename, validation_type) to verification function
_CRYPTO_VERIFIERS: dict[str, Callable] = {

View File

@@ -1,4 +1,4 @@
"""Deduplicate bios/ directory keep one canonical file per unique content.
"""Deduplicate bios/ directory -keep one canonical file per unique content.
Usage:
python scripts/dedup.py [--dry-run] [--bios-dir bios]
@@ -11,7 +11,7 @@ Two types of deduplication:
2. MAME DEVICE CLONES: Different filenames with identical content in the same
MAME directory (e.g., bbc_m87.zip and bbc_24bbc.zip are identical ZIPs).
These are NOT aliases MAME loads each by its unique name. Instead of
These are NOT aliases -MAME loads each by its unique name. Instead of
deleting, we create a _mame_clones.json mapping so generate_pack.py can
pack all names from a single canonical file.
@@ -94,7 +94,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
if len(paths) <= 1:
continue
# Separate by filename same name = true duplicate, different name = clone
# Separate by filename -same name = true duplicate, different name = clone
by_name: dict[str, list[str]] = defaultdict(list)
for p in paths:
by_name[os.path.basename(p)].append(p)
@@ -106,7 +106,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
name_paths.sort(key=path_priority)
true_dupes_to_remove.extend(name_paths[1:])
# Different filenames, same content need special handling
# Different filenames, same content -need special handling
unique_names = sorted(by_name.keys())
if len(unique_names) > 1:
# Check if these are all in MAME/Arcade dirs AND all ZIPs
@@ -133,7 +133,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
true_dupes_to_remove.append(p)
else:
# Non-MAME different names (e.g., 64DD_IPL_US.n64 vs IPL_USA.n64)
# Keep ALL each name may be needed by a different emulator
# Keep ALL -each name may be needed by a different emulator
# Only remove true duplicates (same name in multiple dirs)
pass
@@ -143,7 +143,7 @@ def deduplicate(bios_dir: str, dry_run: bool = False) -> dict:
# Find the best canonical across all paths
all_paths = [p for p in paths if p not in true_dupes_to_remove]
if not all_paths:
# All copies were marked for removal keep the best one
# All copies were marked for removal -keep the best one
all_paths_sorted = sorted(paths, key=path_priority)
all_paths = [all_paths_sorted[0]]
true_dupes_to_remove = [p for p in paths if p != all_paths[0]]

View File

@@ -1,7 +1,7 @@
"""Deterministic ZIP builder for MAME BIOS archives.
Creates byte-identical ZIP files from individual ROM atoms, enabling:
- Reproducible builds: same ROMs same ZIP hash, always
- Reproducible builds: same ROMs -> same ZIP hash, always
- Version-agnostic assembly: build neogeo.zip for any MAME version
- Deduplication: store ROM atoms once, assemble any ZIP on demand

View File

@@ -141,8 +141,8 @@ def scan_bios_dir(bios_dir: Path, cache: dict, force: bool) -> tuple[dict, dict,
def _path_suffix(rel_path: str) -> str:
"""Extract the path suffix after bios/Manufacturer/Console/.
bios/Nintendo/GameCube/GC/USA/IPL.bin GC/USA/IPL.bin
bios/Sony/PlayStation/scph5501.bin scph5501.bin
bios/Nintendo/GameCube/GC/USA/IPL.bin -> GC/USA/IPL.bin
bios/Sony/PlayStation/scph5501.bin -> scph5501.bin
"""
parts = rel_path.replace("\\", "/").split("/")
# Skip: bios / Manufacturer / Console (3 segments)

View File

@@ -248,7 +248,7 @@ def resolve_file(file_entry: dict, db: dict, bios_dir: str,
if path and status != "hash_mismatch":
return path, status
# Large files from GitHub release assets tried when local file is
# Large files from GitHub release assets -tried when local file is
# missing OR has a hash mismatch (wrong variant on disk)
name = file_entry.get("name", "")
sha1 = file_entry.get("sha1")
@@ -362,7 +362,7 @@ def _collect_emulator_extras(
# Second pass: find alternative destinations for files already in the pack.
# A file declared by the platform or emitted above may also be needed at a
# different path by another core (e.g. neocd/ vs root, same_cdi/bios/ vs root).
# Only adds a copy when the file is ALREADY covered at a different path
# Only adds a copy when the file is ALREADY covered at a different path -
# never introduces a file that wasn't selected by the first pass.
profiles = emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir)
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
@@ -831,7 +831,7 @@ def generate_pack(
# Emulator-level validation: informational only for platform packs.
# Platform verification (existence/md5) is the authority for pack status.
# Emulator checks are supplementary logged but don't downgrade.
# Emulator checks are supplementary -logged but don't downgrade.
# When a discrepancy is found, try to find a file satisfying both.
if (file_status.get(dedup_key) == "ok"
and local_path and validation_index):
@@ -892,7 +892,7 @@ def generate_pack(
if base_dest:
full_dest = f"{base_dest}/{dest}"
elif "/" not in dest:
# Bare filename with empty base_destination infer bios/ prefix
# Bare filename with empty base_destination -infer bios/ prefix
# to match platform conventions (RetroDECK: ~/retrodeck/bios/)
full_dest = f"bios/{dest}"
else:
@@ -936,7 +936,7 @@ def generate_pack(
continue
local_path = entry.get("local_cache", "")
if not local_path or not os.path.isdir(local_path):
print(f" WARNING: data directory '{ref_key}' not cached at {local_path} run refresh_data_dirs.py")
print(f" WARNING: data directory '{ref_key}' not cached at {local_path} -run refresh_data_dirs.py")
continue
dd_dest = dd.get("destination", "")
if base_dest and dd_dest:
@@ -961,7 +961,7 @@ def generate_pack(
zf.write(src, full)
total_files += 1
# README.txt for users personalized step-by-step per platform
# README.txt for users -personalized step-by-step per platform
num_systems = len(pack_systems)
readme_text = _build_readme(platform_name, platform_display,
base_dest, total_files, num_systems)
@@ -983,7 +983,7 @@ def generate_pack(
for key, reason in sorted(file_reasons.items()):
status = file_status.get(key, "")
label = "UNTESTED" if status == "untested" else "DISCREPANCY"
print(f" {label}: {key} {reason}")
print(f" {label}: {key} -{reason}")
for name in missing_files:
print(f" MISSING: {name}")
return zip_path
@@ -1011,7 +1011,7 @@ def _normalize_zip_for_pack(source_zip: str, dest_path: str, target_zf: zipfile.
the normalized version into the pack.
This ensures:
- Same ROMs same ZIP hash in every pack build
- Same ROMs -> same ZIP hash in every pack build
- No dependency on how the user built their MAME ROM set
- Bit-identical ZIPs across platforms and build times
"""
@@ -1025,9 +1025,7 @@ def _normalize_zip_for_pack(source_zip: str, dest_path: str, target_zf: zipfile.
os.unlink(tmp_path)
# ---------------------------------------------------------------------------
# Emulator/system mode pack generation
# ---------------------------------------------------------------------------
def _resolve_destination(file_entry: dict, pack_structure: dict | None,
standalone: bool) -> str:
@@ -1081,11 +1079,11 @@ def generate_emulator_pack(
p = all_profiles[name]
if p.get("type") == "alias":
alias_of = p.get("alias_of", "?")
print(f"Error: {name} is an alias of {alias_of} use --emulator {alias_of}",
print(f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}",
file=sys.stderr)
return None
if p.get("type") == "launcher":
print(f"Error: {name} is a launcher use the emulator it launches",
print(f"Error: {name} is a launcher -use the emulator it launches",
file=sys.stderr)
return None
ptype = p.get("type", "libretro")
@@ -1931,9 +1929,7 @@ def main():
emu_profiles, target_cores_cache, system_filter)
# ---------------------------------------------------------------------------
# Manifest generation (JSON inventory for install.py)
# ---------------------------------------------------------------------------
_GITIGNORE_ENTRIES: set[str] | None = None
@@ -2139,7 +2135,7 @@ def generate_manifest(
if case_insensitive:
seen_lower.add(full_dest.lower())
# No phase 3 (data directories) skipped for manifest
# No phase 3 (data directories) -skipped for manifest
now = __import__("datetime").datetime.now(
__import__("datetime").timezone.utc
@@ -2161,9 +2157,7 @@ def generate_manifest(
return result
# ---------------------------------------------------------------------------
# Post-generation pack verification + manifest + SHA256SUMS
# ---------------------------------------------------------------------------
def verify_pack(zip_path: str, db: dict,
data_registry: dict | None = None) -> tuple[bool, dict]:
@@ -2462,7 +2456,7 @@ def verify_pack_against_platform(
if full in zip_set or full.lower() in zip_lower:
core_present += 1
# Not an error if missing some get deduped or filtered
# Not an error if missing -some get deduped or filtered
checked = baseline_checked + core_checked
present = baseline_present + core_present

View File

@@ -189,9 +189,7 @@ def _status_icon(pct: float) -> str:
return "partial"
# ---------------------------------------------------------------------------
# Home page
# ---------------------------------------------------------------------------
def generate_home(db: dict, coverages: dict, profiles: dict,
registry: dict | None = None) -> str:
@@ -303,9 +301,7 @@ def generate_home(db: dict, coverages: dict, profiles: dict,
return "\n".join(lines) + "\n"
# ---------------------------------------------------------------------------
# Platform pages
# ---------------------------------------------------------------------------
def generate_platform_index(coverages: dict) -> str:
lines = [
@@ -478,9 +474,7 @@ def generate_platform_page(name: str, cov: dict, registry: dict | None = None,
return "\n".join(lines) + "\n"
# ---------------------------------------------------------------------------
# System pages
# ---------------------------------------------------------------------------
def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]:
"""Group files by manufacturer -> console -> files."""
@@ -572,9 +566,7 @@ def generate_system_page(
return "\n".join(lines) + "\n"
# ---------------------------------------------------------------------------
# Emulator pages
# ---------------------------------------------------------------------------
def generate_emulators_index(profiles: dict) -> str:
unique = {k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test")}
@@ -1011,9 +1003,7 @@ def generate_emulator_page(name: str, profile: dict, db: dict,
return "\n".join(lines) + "\n"
# ---------------------------------------------------------------------------
# Contributing page
# ---------------------------------------------------------------------------
def generate_gap_analysis(
profiles: dict,
@@ -1367,9 +1357,7 @@ The CI automatically:
"""
# ---------------------------------------------------------------------------
# Wiki pages
# ---------------------------------------------------------------------------
def generate_wiki_index() -> str:
"""Generate wiki landing page."""
@@ -1924,9 +1912,7 @@ def generate_wiki_data_model(db: dict, profiles: dict) -> str:
return "\n".join(lines) + "\n"
# ---------------------------------------------------------------------------
# Build cross-reference indexes
# ---------------------------------------------------------------------------
def _build_platform_file_index(coverages: dict) -> dict[str, set]:
"""Map platform_name -> set of declared file names."""
@@ -1954,9 +1940,7 @@ def _build_emulator_file_index(profiles: dict) -> dict[str, dict]:
return index
# ---------------------------------------------------------------------------
# mkdocs.yml nav generator
# ---------------------------------------------------------------------------
def generate_mkdocs_nav(
coverages: dict,
@@ -2028,9 +2012,7 @@ def generate_mkdocs_nav(
]
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
def main():
parser = argparse.ArgumentParser(description="Generate MkDocs site from project data")
@@ -2053,14 +2035,12 @@ def main():
for d in GENERATED_DIRS:
(docs / d).mkdir(parents=True, exist_ok=True)
# Load registry for platform metadata (logos, etc.)
registry_path = Path(args.platforms_dir) / "_registry.yml"
registry = {}
if registry_path.exists():
with open(registry_path) as f:
registry = (yaml.safe_load(f) or {}).get("platforms", {})
# Load platform configs
platform_names = list_registered_platforms(args.platforms_dir, include_archived=True)
print("Computing platform coverage...")
@@ -2073,7 +2053,6 @@ def main():
except FileNotFoundError as e:
print(f" {name}: skipped ({e})", file=sys.stderr)
# Load emulator profiles
print("Loading emulator profiles...")
profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
unique_count = sum(1 for p in profiles.values() if p.get("type") != "alias")

View File

@@ -24,12 +24,12 @@ SOURCE_URL = (
# Libretro cores that expect BIOS files in a subdirectory of system/.
# System.dat lists filenames flat; the scraper prepends the prefix.
# ref: each core's libretro.c or equivalent see platforms/README.md
# ref: each core's libretro.c or equivalent -see platforms/README.md
CORE_SUBDIR_MAP = {
"nec-pc-98": "np2kai", # libretro-np2kai/sdl/libretro.c
"sharp-x68000": "keropi", # px68k/libretro/libretro.c
"sega-dreamcast": "dc", # flycast/shell/libretro/libretro.cpp
"sega-dreamcast-arcade": "dc", # flycast same subfolder
"sega-dreamcast-arcade": "dc", # flycast -same subfolder
}
SYSTEM_SLUG_MAP = {
@@ -254,7 +254,7 @@ class Scraper(BaseScraper):
systems[req.system]["files"].append(entry)
# Systems not in System.dat but needed for RetroArch added via
# Systems not in System.dat but needed for RetroArch -added via
# shared groups in _shared.yml. The includes directive is resolved
# at load time by load_platform_config().
EXTRA_SYSTEMS = {
@@ -264,7 +264,7 @@ class Scraper(BaseScraper):
"manufacturer": "NEC",
"docs": "https://docs.libretro.com/library/quasi88/",
},
# ref: Vircon32/libretro.c virtual console, single BIOS
# ref: Vircon32/libretro.c -virtual console, single BIOS
"vircon32": {
"files": [
{"name": "Vircon32Bios.v32", "destination": "Vircon32Bios.v32", "required": True},
@@ -273,7 +273,7 @@ class Scraper(BaseScraper):
"manufacturer": "Vircon",
"docs": "https://docs.libretro.com/library/vircon32/",
},
# ref: xrick/src/sysvid.c, xrick/src/data.c game data archive
# ref: xrick/src/sysvid.c, xrick/src/data.c -game data archive
"xrick": {
"files": [
{"name": "data.zip", "destination": "xrick/data.zip", "required": True},
@@ -290,7 +290,7 @@ class Scraper(BaseScraper):
# Arcade BIOS present in the repo but absent from System.dat.
# FBNeo expects them in system/ or system/fbneo/.
# ref: fbneo/src/burner/libretro/libretro.cpp
# ref: fbneo/src/burner/libretro/libretro.cpp search order:
# ref: fbneo/src/burner/libretro/libretro.cpp -search order:
# 1) romset dir 2) system/fbneo/ 3) system/
EXTRA_ARCADE_FILES = [
{"name": "namcoc69.zip", "destination": "namcoc69.zip", "required": True},
@@ -329,33 +329,33 @@ class Scraper(BaseScraper):
# Extra files missing from System.dat for specific systems.
# Each traced to the core's source code.
EXTRA_SYSTEM_FILES = {
# melonDS DS DSi mode ref: JesseTG/melonds-ds/src/libretro.cpp
# melonDS DS DSi mode -ref: JesseTG/melonds-ds/src/libretro.cpp
"nintendo-ds": [
{"name": "dsi_bios7.bin", "destination": "dsi_bios7.bin", "required": True},
{"name": "dsi_bios9.bin", "destination": "dsi_bios9.bin", "required": True},
{"name": "dsi_firmware.bin", "destination": "dsi_firmware.bin", "required": True},
{"name": "dsi_nand.bin", "destination": "dsi_nand.bin", "required": True},
],
# bsnes SGB naming ref: bsnes/target-libretro/libretro.cpp
# bsnes SGB naming -ref: bsnes/target-libretro/libretro.cpp
"nintendo-sgb": [
{"name": "sgb.boot.rom", "destination": "sgb.boot.rom", "required": False},
],
# JollyCV ref: jollycv/libretro.c
# JollyCV -ref: jollycv/libretro.c
"coleco-colecovision": [
{"name": "BIOS.col", "destination": "BIOS.col", "required": True},
{"name": "coleco.rom", "destination": "coleco.rom", "required": True},
{"name": "bioscv.rom", "destination": "bioscv.rom", "required": True},
],
# Kronos ST-V ref: libretro-kronos/libretro/libretro.c
# Kronos ST-V -ref: libretro-kronos/libretro/libretro.c
"sega-saturn": [
{"name": "stvbios.zip", "destination": "kronos/stvbios.zip", "required": True},
],
# PCSX ReARMed / Beetle PSX alt BIOS ref: pcsx_rearmed/libpcsxcore/misc.c
# PCSX ReARMed / Beetle PSX alt BIOS -ref: pcsx_rearmed/libpcsxcore/misc.c
# docs say PSXONPSP660.bin (uppercase) but core accepts any case
"sony-playstation": [
{"name": "psxonpsp660.bin", "destination": "psxonpsp660.bin", "required": False},
],
# Dolphin GC ref: DolphinLibretro/Boot.cpp:72-73,
# Dolphin GC -ref: DolphinLibretro/Boot.cpp:72-73,
# BootManager.cpp:200-217, CommonPaths.h:139 GC_IPL="IPL.bin"
# Core searches system/dolphin-emu/Sys/ for data and BIOS.
# System.dat gc-ntsc-*.bin names are NOT what Dolphin loads.
@@ -364,15 +364,15 @@ class Scraper(BaseScraper):
{"name": "gc-ntsc-12.bin", "destination": "dolphin-emu/Sys/GC/USA/IPL.bin", "required": False},
{"name": "gc-pal-12.bin", "destination": "dolphin-emu/Sys/GC/EUR/IPL.bin", "required": False},
{"name": "gc-ntsc-12.bin", "destination": "dolphin-emu/Sys/GC/JAP/IPL.bin", "required": False},
# DSP firmware ref: Source/Core/Core/HW/DSPLLE/DSPHost.cpp
# DSP firmware -ref: Source/Core/Core/HW/DSPLLE/DSPHost.cpp
{"name": "dsp_coef.bin", "destination": "dolphin-emu/Sys/GC/dsp_coef.bin", "required": True},
{"name": "dsp_rom.bin", "destination": "dolphin-emu/Sys/GC/dsp_rom.bin", "required": True},
# Fonts ref: Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp
# Fonts -ref: Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp
{"name": "font_western.bin", "destination": "dolphin-emu/Sys/GC/font_western.bin", "required": False},
{"name": "font_japanese.bin", "destination": "dolphin-emu/Sys/GC/font_japanese.bin", "required": False},
],
# minivmac casing ref: minivmac/src/MYOSGLUE.c
# doc says MacII.rom, repo has MacII.ROM both work on case-insensitive FS
# minivmac casing -ref: minivmac/src/MYOSGLUE.c
# doc says MacII.rom, repo has MacII.ROM -both work on case-insensitive FS
"apple-macintosh-ii": [
{"name": "MacII.ROM", "destination": "MacII.ROM", "required": True},
],
@@ -398,7 +398,7 @@ class Scraper(BaseScraper):
# Inject shared group references for systems that have core-specific
# subdirectory requirements already defined in _shared.yml.
# Note: fuse/ prefix NOT injected for sinclair-zx-spectrum.
# Verified in fuse-libretro/src/compat/paths.c core searches
# Verified in fuse-libretro/src/compat/paths.c -core searches
# system/ flat, not fuse/ subfolder. Docs are wrong on this.
SYSTEM_SHARED_GROUPS = {
"nec-pc-98": ["np2kai"],
@@ -421,12 +421,12 @@ class Scraper(BaseScraper):
{"ref": "ppsspp-assets", "destination": "PPSSPP"},
],
# single buildbot ZIP contains both Databases/ and Machines/
# ref: libretro.c:1118-1119 system_dir/Machines + system_dir/Databases
# ref: libretro.c:1118-1119 -system_dir/Machines + system_dir/Databases
"microsoft-msx": [
{"ref": "bluemsx", "destination": ""},
],
# FreeIntv overlays system/freeintv_overlays/<rom>.png
# ref: FreeIntv/src/libretro.c:273 stbi_load from system dir
# FreeIntv overlays -system/freeintv_overlays/<rom>.png
# ref: FreeIntv/src/libretro.c:273 -stbi_load from system dir
# ZIP contains FreeIntvTS_Overlays/ subfolder, cache preserves it
# pack destination maps cache root to system/freeintv_overlays
# so final path is system/freeintv_overlays/FreeIntvTS_Overlays/<rom>.png

View File

@@ -171,7 +171,7 @@ def _resolve_path(p: str) -> str:
def _extract_bios_entries(component_val: dict) -> list[dict]:
"""Extract BIOS entries from all three possible locations in a component.
No dedup here dedup is done in fetch_requirements() with full
No dedup here -dedup is done in fetch_requirements() with full
(system, filename) key to avoid dropping valid same-filename entries
across different systems.
"""
@@ -338,13 +338,13 @@ class Scraper(BaseScraper):
if resolved.startswith("saves"):
continue
# Build destination default to bios/ if no path specified
# Build destination -default to bios/ if no path specified
if resolved:
destination = f"{resolved}/{filename}"
else:
destination = f"bios/{filename}"
# MD5 handling sanitize upstream errors
# MD5 handling -sanitize upstream errors
md5_raw = entry.get("md5", "")
if isinstance(md5_raw, list):
parts = [str(m).strip().lower() for m in md5_raw if m]

View File

@@ -136,7 +136,7 @@ def _check_atom(tokens: list[str], pos: int, active: frozenset[str]) -> tuple[bo
if tok.startswith('"'):
pos += 1
return True, pos
# Unknown token treat as true to avoid false negatives
# Unknown token -treat as true to avoid false negatives
pos += 1
return True, pos

View File

@@ -1,8 +1,8 @@
"""Scraper for EmuDeck emulator targets.
Sources:
SteamOS: dragoonDorise/EmuDeck functions/EmuScripts/*.sh
Windows: EmuDeck/emudeck-we functions/EmuScripts/*.ps1
SteamOS: dragoonDorise/EmuDeck -functions/EmuScripts/*.sh
Windows: EmuDeck/emudeck-we -functions/EmuScripts/*.ps1
"""
from __future__ import annotations

View File

@@ -56,7 +56,7 @@ TARGETS: list[tuple[str, str, str]] = [
("nintendo/wiiu/latest", "nintendo-wiiu", "ppc"),
("playstation/ps2/latest", "playstation-ps2", "mips"),
("playstation/psp/latest", "playstation-psp", "mips"),
# vita: only VPK bundles on buildbot cores listed via libretro-super recipes
# vita: only VPK bundles on buildbot -cores listed via libretro-super recipes
]
# Recipe-based targets: (recipe_path_under_RECIPE_BASE_URL, target_name, architecture)

View File

@@ -3,7 +3,7 @@
Implements GF(2^233) field arithmetic, elliptic curve point operations,
and ECDSA-SHA256 verification for Nintendo 3DS OTP certificate checking.
Zero external dependencies uses only Python stdlib.
Zero external dependencies -uses only Python stdlib.
Curve: sect233r1 (NIST B-233, SEC 2 v2)
Field: GF(2^233) with irreducible polynomial t^233 + t^74 + 1
@@ -13,9 +13,7 @@ from __future__ import annotations
import hashlib
# ---------------------------------------------------------------------------
# sect233r1 curve parameters (SEC 2 v2)
# ---------------------------------------------------------------------------
_M = 233
_F = (1 << 233) | (1 << 74) | 1 # irreducible polynomial
@@ -34,9 +32,7 @@ _N_BITLEN = _N.bit_length() # 233
_H = 2
# ---------------------------------------------------------------------------
# GF(2^233) field arithmetic
# ---------------------------------------------------------------------------
def _gf_reduce(a: int) -> int:
"""Reduce polynomial a modulo t^233 + t^74 + 1."""
@@ -85,7 +81,7 @@ def _gf_inv(a: int) -> int:
q ^= 1 << shift
temp ^= r << shift
remainder = temp
# Multiply q * s in GF(2)[x] (no reduction working in polynomial ring)
# Multiply q * s in GF(2)[x] (no reduction -working in polynomial ring)
qs = 0
qt = q
st = s
@@ -102,10 +98,8 @@ def _gf_inv(a: int) -> int:
return _gf_reduce(old_s)
# ---------------------------------------------------------------------------
# Elliptic curve point operations on sect233r1
# y^2 + xy = x^3 + ax^2 + b (a=1)
# ---------------------------------------------------------------------------
# Point at infinity
_INF = None
@@ -175,9 +169,7 @@ def _ec_mul(k: int, p: tuple[int, int] | None) -> tuple[int, int] | None:
return result
# ---------------------------------------------------------------------------
# ECDSA-SHA256 verification
# ---------------------------------------------------------------------------
def _modinv(a: int, m: int) -> int:
"""Modular inverse of a modulo m (integers, not GF(2^m))."""

View File

@@ -217,7 +217,7 @@ def generate_platform_truth(
sys_cov["profiled"].add(emu_name)
# Ensure all systems of resolved cores have entries (even with 0 files).
# This documents that the system is covered the core was analyzed and
# This documents that the system is covered -the core was analyzed and
# needs no external files for this system.
for emu_name in cores_profiled:
profile = profiles[emu_name]
@@ -261,9 +261,7 @@ def generate_platform_truth(
}
# -------------------------------------------------------------------
# Platform truth diffing
# -------------------------------------------------------------------
def _diff_system(truth_sys: dict, scraped_sys: dict) -> dict:
"""Compare files between truth and scraped for a single system."""
@@ -430,7 +428,7 @@ def diff_platform_truth(truth: dict, scraped: dict) -> dict:
else:
summary["systems_fully_covered"] += 1
# Truth systems not matched by any scraped system all files missing
# Truth systems not matched by any scraped system -all files missing
for t_sid in sorted(truth_systems):
if t_sid in matched_truth:
continue

View File

@@ -12,7 +12,7 @@ import os
from common import compute_hashes
# Validation types that require console-specific cryptographic keys.
# verify.py cannot reproduce these size checks still apply if combined.
# verify.py cannot reproduce these -size checks still apply if combined.
_CRYPTO_CHECKS = frozenset({"signature", "crypto"})
# All reproducible validation types.
@@ -85,7 +85,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
if f.get("max_size") is not None:
cur = index[fname]["max_size"]
index[fname]["max_size"] = max(cur, f["max_size"]) if cur is not None else f["max_size"]
# Hash checks collect all accepted hashes as sets (multiple valid
# Hash checks -collect all accepted hashes as sets (multiple valid
# versions of the same file, e.g. MT-32 ROM versions)
if "crc32" in checks and f.get("crc32"):
crc_val = f["crc32"]
@@ -103,7 +103,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]:
index[fname][hash_type].add(str(h).lower())
else:
index[fname][hash_type].add(str(val).lower())
# Adler32 stored as known_hash_adler32 field (not in validation: list
# Adler32 -stored as known_hash_adler32 field (not in validation: list
# for Dolphin, but support it in both forms for future profiles)
adler_val = f.get("known_hash_adler32") or f.get("adler32")
if adler_val:
@@ -186,7 +186,7 @@ def check_file_validation(
return None
checks = entry["checks"]
# Size checks sizes is a set of accepted values
# Size checks -sizes is a set of accepted values
if "size" in checks:
actual_size = os.path.getsize(local_path)
if entry["sizes"] and actual_size not in entry["sizes"]:
@@ -197,7 +197,7 @@ def check_file_validation(
if entry["max_size"] is not None and actual_size > entry["max_size"]:
return f"size too large: max {entry['max_size']}, got {actual_size}"
# Hash checks compute once, reuse for all hash types.
# Hash checks -compute once, reuse for all hash types.
# Each hash field is a set of accepted values (multiple valid ROM versions).
need_hashes = (
any(h in checks and entry.get(h) for h in ("crc32", "md5", "sha1", "sha256"))

View File

@@ -47,9 +47,7 @@ DEFAULT_PLATFORMS_DIR = "platforms"
DEFAULT_EMULATORS_DIR = "emulators"
# ---------------------------------------------------------------------------
# Status model — aligned with Batocera BiosStatus (batocera-systems:967-969)
# ---------------------------------------------------------------------------
# Status model -aligned with Batocera BiosStatus (batocera-systems:967-969)
class Status:
OK = "ok"
@@ -68,15 +66,13 @@ _STATUS_ORDER = {Status.OK: 0, Status.UNTESTED: 1, Status.MISSING: 2}
_SEVERITY_ORDER = {Severity.OK: 0, Severity.INFO: 1, Severity.WARNING: 2, Severity.CRITICAL: 3}
# ---------------------------------------------------------------------------
# Verification functions
# ---------------------------------------------------------------------------
def verify_entry_existence(
file_entry: dict, local_path: str | None,
validation_index: dict[str, dict] | None = None,
) -> dict:
"""RetroArch verification: path_is_valid() file exists = OK."""
"""RetroArch verification: path_is_valid() -file exists = OK."""
name = file_entry.get("name", "")
required = file_entry.get("required", True)
if not local_path:
@@ -96,7 +92,7 @@ def verify_entry_md5(
local_path: str | None,
resolve_status: str = "",
) -> dict:
"""MD5 verification Batocera md5sum + Recalbox multi-hash + Md5Composite."""
"""MD5 verification -Batocera md5sum + Recalbox multi-hash + Md5Composite."""
name = file_entry.get("name", "")
expected_md5 = file_entry.get("md5", "")
zipped_file = file_entry.get("zipped_file")
@@ -162,7 +158,7 @@ def verify_entry_sha1(
file_entry: dict,
local_path: str | None,
) -> dict:
"""SHA1 verification BizHawk firmware hash check."""
"""SHA1 verification -BizHawk firmware hash check."""
name = file_entry.get("name", "")
expected_sha1 = file_entry.get("sha1", "")
required = file_entry.get("required", True)
@@ -183,20 +179,18 @@ def verify_entry_sha1(
"reason": f"expected {expected_sha1[:12]}… got {actual_sha1[:12]}"}
# ---------------------------------------------------------------------------
# Severity mapping per platform
# ---------------------------------------------------------------------------
def compute_severity(
status: str, required: bool, mode: str, hle_fallback: bool = False,
) -> str:
"""Map (status, required, verification_mode, hle_fallback) severity.
"""Map (status, required, verification_mode, hle_fallback) -> severity.
Based on native platform behavior + emulator HLE capability:
- RetroArch (existence): required+missing = warning, optional+missing = info
- Batocera/Recalbox/RetroBat/EmuDeck (md5): hash-based verification
- BizHawk (sha1): same severity rules as md5
- hle_fallback: core works without this file via HLE always INFO when missing
- hle_fallback: core works without this file via HLE -> always INFO when missing
"""
if status == Status.OK:
return Severity.OK
@@ -218,13 +212,9 @@ def compute_severity(
return Severity.OK
# ---------------------------------------------------------------------------
# ZIP content index
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Cross-reference: undeclared files used by cores
# ---------------------------------------------------------------------------
def _build_expected(file_entry: dict, checks: list[str]) -> dict:
@@ -447,7 +437,7 @@ def find_exclusion_notes(
})
continue
# Count standalone-only files but only report as excluded if the
# Count standalone-only files -but only report as excluded if the
# platform does NOT use this emulator in standalone mode
standalone_set = set(str(c) for c in config.get("standalone_cores", []))
is_standalone = emu_name in standalone_set or bool(
@@ -466,9 +456,7 @@ def find_exclusion_notes(
return notes
# ---------------------------------------------------------------------------
# Platform verification
# ---------------------------------------------------------------------------
def _find_best_variant(
file_entry: dict, db: dict, current_path: str,
@@ -640,9 +628,7 @@ def verify_platform(
}
# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------
def _format_ground_truth_aggregate(ground_truth: list[dict]) -> str:
"""Format ground truth as a single aggregated line.
@@ -698,7 +684,7 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) ->
req = "required" if d.get("required", True) else "optional"
hle = ", HLE available" if d.get("hle_fallback") else ""
reason = d.get("reason", "")
print(f" UNTESTED ({req}{hle}): {key} {reason}")
print(f" UNTESTED ({req}{hle}): {key} -{reason}")
_print_ground_truth(d.get("ground_truth", []), verbose)
for d in details:
if d["status"] == Status.MISSING:
@@ -717,7 +703,7 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) ->
if key in seen:
continue
seen.add(key)
print(f" DISCREPANCY: {key} {disc}")
print(f" DISCREPANCY: {key} -{disc}")
_print_ground_truth(d.get("ground_truth", []), verbose)
if verbose:
@@ -831,7 +817,7 @@ def print_platform_result(result: dict, group: list[str], verbose: bool = False)
if exclusions:
print(f" No external files ({len(exclusions)}):")
for ex in exclusions:
print(f" {ex['emulator']} {ex['detail']} [{ex['reason']}]")
print(f" {ex['emulator']} -{ex['detail']} [{ex['reason']}]")
gt_cov = result.get("ground_truth_coverage")
if gt_cov and gt_cov["total"] > 0:
@@ -841,9 +827,7 @@ def print_platform_result(result: dict, group: list[str], verbose: bool = False)
print(f" {gt_cov['platform_only']} platform-only (no emulator profile)")
# ---------------------------------------------------------------------------
# Emulator/system mode verification
# ---------------------------------------------------------------------------
def _effective_validation_label(details: list[dict], validation_index: dict) -> str:
"""Determine the bracket label for the report.
@@ -892,11 +876,11 @@ def verify_emulator(
p = all_profiles[name]
if p.get("type") == "alias":
alias_of = p.get("alias_of", "?")
print(f"Error: {name} is an alias of {alias_of} use --emulator {alias_of}",
print(f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}",
file=sys.stderr)
sys.exit(1)
if p.get("type") == "launcher":
print(f"Error: {name} is a launcher use the emulator it launches",
print(f"Error: {name} is a launcher -use the emulator it launches",
file=sys.stderr)
sys.exit(1)
# Check standalone capability
@@ -1117,7 +1101,7 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None:
req = "required" if d.get("required", True) else "optional"
hle = ", HLE available" if d.get("hle_fallback") else ""
reason = d.get("reason", "")
print(f" UNTESTED ({req}{hle}): {d['name']} {reason}")
print(f" UNTESTED ({req}{hle}): {d['name']} -{reason}")
gt = d.get("ground_truth", [])
if gt:
if verbose: