#!/usr/bin/env python3 """Generate platform-specific BIOS ZIP packs. Usage: python scripts/generate_pack.py --platform retroarch [--output-dir dist/] python scripts/generate_pack.py --all [--output-dir dist/] Reads platform YAML config + database.json -> creates ZIP with correct file layout for each platform. Handles inheritance, shared groups, variants, and 3-tier storage (embedded/external/user_provided). """ from __future__ import annotations import argparse import contextlib import hashlib import json import os import re import sys import tempfile import urllib.error import urllib.request import zipfile from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from common import ( artifact_lock, ArtifactLockBusy, build_target_cores_cache, build_zip_contents_index, check_inside_zip, compute_hashes, expand_platform_declared_names, fetch_large_file, filter_systems_by_target, group_identical_platforms, list_emulator_profiles, list_platform_system_ids, list_registered_platforms, list_system_ids, load_data_dir_registry, load_database, load_emulator_profiles, load_platform_config, MANUFACTURER_PREFIXES, parse_md5_list, require_yaml, resolution_is_hash_exact, resolve_local_file, yaml_load, ) import region as region_mod import slot as slot_mod from deterministic_zip import _FIXED_DATE_TIME, rebuild_zip_deterministic from nativemode import hash_mismatch_excludes_file from validation import ( _build_validation_index, check_file_validation, filter_files_by_mode, ) yaml = require_yaml() DEFAULT_PLATFORMS_DIR = "platforms" DEFAULT_DB_FILE = "database.json" DEFAULT_OUTPUT_DIR = "dist" DEFAULT_BIOS_DIR = "bios" # CLI-wide network policy. Public helpers can override it explicitly, while # every CLI mode inherits --offline (including large-file release fallback). _OFFLINE = False _HEX_RE = re.compile(r"\b([0-9a-fA-F]{8,40})\b") def _write_generated_member(zf: zipfile.ZipFile, arcname: str, text: str) -> None: """Write a pack member the builder composes rather than copies. READMEs, instruction notes and the manifest are produced at build time, so writestr would stamp them with the wall clock and give the same pack a different hash on every run. Pinning them to the epoch the archive rebuilder already uses makes a pack a function of its inputs alone. """ info = zipfile.ZipInfo(filename=arcname, date_time=_FIXED_DATE_TIME) info.compress_type = zipfile.ZIP_DEFLATED info.external_attr = 0o100644 << 16 zf.writestr(info, text) def _build_timestamp(db: dict | None = None) -> str: """Timestamp for generated artifacts. Reads the database snapshot the artifact was built from, so rebuilding the same data twice yields the same value. Falls back to the clock only when no database is at hand. """ stamp = (db or {}).get("generated_at") if isinstance(stamp, str) and stamp: return stamp from datetime import datetime, timezone return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") def _detect_hash_type(h: str) -> str: n = len(h) if n == 40: return "sha1" if n == 32: return "md5" if n == 8: return "crc32" return "md5" def parse_hash_input(raw: str) -> list[tuple[str, str]]: """Parse comma-separated hash string into (type, hash) tuples.""" results: list[tuple[str, str]] = [] for part in raw.split(","): part = part.strip().lower() if not part: continue m = _HEX_RE.search(part) if m: h = m.group(1) results.append((_detect_hash_type(h), h)) return results def parse_hash_file(path: str) -> list[tuple[str, str]]: """Parse hash file (one per line, comments with #, mixed formats).""" results: list[tuple[str, str]] = [] with open(path) as f: for line in f: line = line.strip() if not line or line.startswith("#"): continue m = _HEX_RE.search(line.lower()) if m: h = m.group(1) results.append((_detect_hash_type(h), h)) return results def lookup_hashes( hashes: list[tuple[str, str]], db: dict, bios_dir: str, emulators_dir: str, platforms_dir: str, ) -> None: """Print diagnostic info for each hash.""" files_db = db.get("files", {}) by_md5 = db.get("indexes", {}).get("by_md5", {}) by_crc32 = db.get("indexes", {}).get("by_crc32", {}) for hash_type, hash_val in hashes: sha1 = None if hash_type == "sha1" and hash_val in files_db: sha1 = hash_val elif hash_type == "md5": sha1 = by_md5.get(hash_val) elif hash_type == "crc32": sha1 = by_crc32.get(hash_val) if not sha1 or sha1 not in files_db: print(f"\n{hash_type.upper()}: {hash_val}") print(" NOT FOUND in database") continue entry = files_db[sha1] name = entry.get("name", "?") md5 = entry.get("md5", "?") paths = entry.get("paths") or [] aliases = entry.get("aliases") or [] print(f"\n{hash_type.upper()}: {hash_val}") print(f" SHA1: {sha1}") print(f" MD5: {md5}") print(f" Name: {name}") if paths: print(f" Path: {paths[0]}") if aliases: print(f" Aliases: {aliases}") # Check if file exists in repo (by path or by resolve_local_file) in_repo = False if paths: primary = os.path.join(bios_dir, paths[0]) if os.path.exists(primary): in_repo = True if not in_repo: try: fe_check = {"name": name, "sha1": sha1, "md5": md5} local, status = resolve_file(fe_check, db, bios_dir, {}) if local and status != "not_found": in_repo = True except (KeyError, OSError): pass print(f" In repo: {'YES' if in_repo else 'NO'}") def _find_candidate_satisfying_both( file_entry: dict, db: dict, local_path: str, validation_index: dict, bios_dir: str, ) -> str | None: """Search for a repo file that satisfies both platform MD5 and emulator validation. When the current file passes platform verification but fails emulator checks, search all candidates with the same name for one that passes both. Returns a better path, or None if no upgrade found. """ fname = file_entry.get("name", "") if not fname: return None entry = validation_index.get(fname) if not entry: return None md5_expected = file_entry.get("md5", "") md5_set = ( {m.strip().lower() for m in md5_expected.split(",") if m.strip()} if md5_expected else set() ) by_name = db.get("indexes", {}).get("by_name", {}) files_db = db.get("files", {}) for sha1 in by_name.get(fname, []): candidate = files_db.get(sha1, {}) path = candidate.get("path", "") if ( not path or not os.path.exists(path) or os.path.realpath(path) == os.path.realpath(local_path) ): continue # Must still satisfy platform MD5 if md5_set and candidate.get("md5", "").lower() not in md5_set: continue # Check emulator validation reason = check_file_validation(path, fname, validation_index, bios_dir) if reason is None: return path return None def _emulator_systems_index(emu_profiles: dict | None) -> dict[str, list[str]]: """Map both the profile key and its display name to the profile's systems. find_undeclared_files reports the display name ("Beetle PSX (Mednafen PSX)"), while the profile dictionary is keyed by slug. A key-only lookup therefore missed almost every core, dropping its files into one shared bucket and losing the per-system grouping the narrowing passes rely on. """ index: dict[str, list[str]] = {} for key, profile in (emu_profiles or {}).items(): systems = list(profile.get("systems", [])) index[key] = systems display = profile.get("emulator", "") if display: index.setdefault(display, systems) return index def _pack_member_groups( config: dict, pack_systems: dict, emulators_dir: str, db: dict, base_dest: str, emu_profiles: dict | None, target_cores: set[str] | None, source: str, ) -> dict[str, list[tuple[str, str]]]: """Group every candidate of a pack by system, as (destination, name). Covers the platform baseline and the core extras, because a narrowing pass that saw only one of the two would compare an incomplete set. """ groups: dict[str, list[tuple[str, str]]] = {} for sys_id, system in pack_systems.items(): members = groups.setdefault(sys_id, []) for fe in system.get("files", []): dest = _sanitize_path(fe.get("destination", fe.get("name", ""))) if dest: members.append((dest, fe.get("name", ""))) if source == "platform": return groups # find_undeclared_files reports the profile's display name, not its key, # so index both: on a key-only lookup nearly every core extra fell into a # single bucket and lost its per-system grouping. emu_systems = _emulator_systems_index(emu_profiles) for fe in _collect_emulator_extras( config, emulators_dir, db, set(), base_dest, emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ): dest = _sanitize_path(fe.get("destination", fe.get("name", ""))) if not dest: continue for sys_id in emu_systems.get(fe.get("source_emulator", ""), ["_extras"]): groups.setdefault(sys_id, []).append((dest, fe.get("name", ""))) return groups def _narrowings( source: str, regions: list[str] | None, target_name: str | None, one_per_slot: bool, required_only: bool, standalone: bool = False, ) -> list[tuple[str, str]]: """Every dimension that narrows a pack, as (filename tag, plain label). The pack name and the pack README are both built from this list, so a new way of narrowing a pack cannot reach users nameless or unannounced. Order is the filename order and must not change: it is what existing archives are called. """ applied: list[tuple[str, str]] = [] if source == "platform": applied.append(("_Platform", "only what the platform declares")) elif source == "truth": applied.append(("_Truth", "only what emulator profiles require")) if regions: pretty = ", ".join( " ".join(w.title() for w in slug.split("-")) for slug in regions ) applied.append( (f"_{region_mod.region_tag(regions)}", f"region priority {pretty}") ) if target_name: applied.append( (f"_{_target_tag(target_name)}", f"cores available on {target_name}") ) if one_per_slot: applied.append(("_OnePerSlot", "one BIOS per system and region")) if standalone: applied.append(("_Standalone", "standalone mode files only")) if required_only: applied.append(("_Required", "required files only")) return applied def _target_tag(target_name: str) -> str: """Filename tag for a hardware target. Without it a targeted pack takes the same name as the full one and overwrites it. """ return "".join( part.title() for part in re.split(r"[-_\s]+", target_name.strip()) if part ) def _sanitize_path(raw: str) -> str: """Strip path traversal components from a relative path.""" raw = raw.replace("\\", "/") parts = [p for p in raw.split("/") if p and p not in ("..", ".")] return "/".join(parts) def _path_parents(dest: str) -> list[str]: """Return all parent directory segments of a path.""" parts = dest.split("/") return ["/".join(parts[:i]) for i in range(1, len(parts))] def _has_path_conflict(dest: str, seen_files: set[str], seen_parents: set[str]) -> bool: """Check if dest conflicts with existing paths (file vs directory). Returns True if adding dest would create an impossible extraction: - A parent of dest is already a file (e.g., adding X/Y when X is a file) - dest itself is already used as a directory (e.g., adding X when X/Y exists) """ for parent in _path_parents(dest): if parent in seen_files: return True if dest in seen_parents: return True return False def _register_path(dest: str, seen_files: set[str], seen_parents: set[str]) -> None: """Track a file path and its parent directories.""" seen_files.add(dest) for parent in _path_parents(dest): seen_parents.add(parent) def _flat(arcname: str, prefix: str, flatten: bool) -> str: """Strip base_destination prefix from ZIP arcname when flattening.""" if flatten and prefix and arcname.startswith(prefix + "/"): return arcname[len(prefix) + 1:] return arcname def resolve_file( file_entry: dict, db: dict, bios_dir: str, zip_contents: dict | None = None, dest_hint: str = "", data_dir_registry: dict | None = None, *, offline: bool | None = None, ) -> tuple[str | None, str]: """Resolve a BIOS file with storage tiers and release asset fallback. Wraps common.resolve_local_file() with pack-specific logic for storage tiers (external/user_provided), large file release assets, and MAME clone mapping (deduped ZIPs). """ storage = file_entry.get("storage", "embedded") if storage == "user_provided": return None, "user_provided" if storage == "external": return None, "external" path, status = resolve_local_file( file_entry, db, zip_contents, dest_hint=dest_hint, data_dir_registry=data_dir_registry, ) if path and status != "hash_mismatch": return path, status # Large files from GitHub release assets -tried when local file is # missing OR has a hash mismatch (wrong variant on disk) name = file_entry.get("name", "") sha1 = file_entry.get("sha1") first_sha1 = (sha1[0] if sha1 else "") if isinstance(sha1, list) else (sha1 or "") md5_list = parse_md5_list(file_entry.get("md5")) first_md5 = md5_list[0] if md5_list else "" cached = fetch_large_file( name, expected_sha1=first_sha1, expected_md5=first_md5, offline=_OFFLINE if offline is None else offline, ) if cached: return cached, "release_asset" # Fall back to hash_mismatch local file if release asset unavailable if path: return path, status return None, "not_found" def download_external(file_entry: dict, dest_path: str) -> bool: """Download an external BIOS file, verify hash, save to dest_path.""" url = file_entry.get("source_url") if not url: return False sha256 = file_entry.get("sha256") sha1 = file_entry.get("sha1") md5 = file_entry.get("md5") if not (sha256 or sha1 or md5): print( f" WARNING: no hash for {file_entry['name']}, skipping unverifiable download" ) return False try: req = urllib.request.Request( url, headers={"User-Agent": "retrobios-pack-gen/1.0"} ) with urllib.request.urlopen(req, timeout=120) as resp: data = resp.read() except urllib.error.URLError as e: print(f" WARNING: Failed to download {url}: {e}") return False if sha256: actual = hashlib.sha256(data).hexdigest() if actual != sha256: print(f" WARNING: SHA256 mismatch for {file_entry['name']}") return False elif sha1: actual = hashlib.sha1(data).hexdigest() if actual != sha1: print(f" WARNING: SHA1 mismatch for {file_entry['name']}") return False elif md5: actual = hashlib.md5(data).hexdigest() if actual != md5: print(f" WARNING: MD5 mismatch for {file_entry['name']}") return False os.makedirs(os.path.dirname(dest_path), exist_ok=True) with open(dest_path, "wb") as f: f.write(data) return True def _detect_extras_prefix(config: dict, base_dest: str) -> str: """Detect the effective BIOS prefix for core extras. When base_destination is empty (RetroDECK), infer the prefix from the dominant root of YAML-declared destinations. Returns the prefix to prepend to every core-extra destination (may be empty). """ if base_dest: return base_dest dests: list[str] = [] for sys_data in config.get("systems", {}).values(): for f in sys_data.get("files", []): d = f.get("destination", "") if d and "/" in d: dests.append(d) if not dests: return "" from collections import Counter roots = Counter(d.split("/", 1)[0] for d in dests) most_common, count = roots.most_common(1)[0] if count / len(dests) > 0.9: return most_common return "" def _detect_slug_structure(config: dict) -> tuple[bool, dict[str, str]]: """Detect whether a platform uses per-system slug destinations. Returns ``(is_slug_based, system_to_slug)`` where ``system_to_slug`` maps system IDs to their destination slug prefix. Slug-based means each system's files live under a per-system subfolder (e.g. RomM's ``bios/{platform_slug}/{file}``), with varying slugs across systems. Only returns True when nearly ALL destinations have a subfolder and nearly ALL systems map to a consistent slug, distinguishing true slug-based layouts (RomM) from platforms that happen to have some subfoldered files (RetroArch ``dc/``, ``neocd/``). """ total_files = 0 files_with_slash = 0 sys_to_slug: dict[str, str] = {} total_systems_with_files = 0 for sys_id, sys_data in config.get("systems", {}).items(): files = sys_data.get("files", []) if not files: continue total_systems_with_files += 1 slugs: set[str] = set() for f in files: d = f.get("destination", "") if d: total_files += 1 if "/" in d: files_with_slash += 1 slugs.add(d.split("/", 1)[0]) if len(slugs) == 1: sys_to_slug[sys_id] = slugs.pop() if not sys_to_slug or total_files == 0: return False, {} # All conditions must hold for slug-based detection: # 1. Nearly all files have a subfolder # 2. Multiple distinct slugs (not a constant prefix) # 3. Nearly all systems with files map to a slug # 4. Files are exactly slug/filename (depth 2), not deeper unique_slugs = set(sys_to_slug.values()) all_have_slash = files_with_slash / total_files > 0.95 varying_slugs = len(unique_slugs) > 1 high_coverage = len(sys_to_slug) / total_systems_with_files > 0.9 # Count files deeper than slug/filename (e.g., amiga/bios/kick.rom) deep_files = 0 for sys_data in config.get("systems", {}).values(): for f in sys_data.get("files", []): d = f.get("destination", "") if d and d.count("/") > 1: deep_files += 1 shallow = deep_files / total_files < 0.05 if total_files else True return (all_have_slash and varying_slugs and high_coverage and shallow), sys_to_slug def _map_emulator_to_slug( profile: dict, platform_systems: set[str], norm_map: dict[str, str], sys_to_slug: dict[str, str], ) -> str: """Map an emulator to a destination slug for slug-based platforms.""" from common import _norm_system_id emu_systems = set(profile.get("systems", [])) # Direct match direct = emu_systems & platform_systems if direct: target = sorted(direct)[0] return sys_to_slug.get(target, "") # Normalized match for es in sorted(emu_systems): norm = _norm_system_id(es) if norm in norm_map: target = norm_map[norm] return sys_to_slug.get(target, "") return "" def _collect_emulator_extras( config: dict, emulators_dir: str, db: dict, seen: set, base_dest: str, emu_profiles: dict | None = None, target_cores: set[str] | None = None, include_all: bool = False, ) -> list[dict]: """Collect core requirement files from emulator profiles not in the platform pack. Uses the same system-overlap matching as verify.py cross-reference: - Matches emulators by shared system IDs with the platform - Filters mode: standalone, type: launcher, type: alias - Respects data_directories coverage - Only returns files that exist in the repo (packable) When the same file is needed at multiple destinations by different cores (e.g. cdimono1.zip at root for cdi2015 and at same_cdi/bios/ for same_cdi), all destinations are included so every core finds its files. Works for ANY platform (RetroArch, Batocera, Recalbox, etc.) """ from common import _norm_system_id, resolve_platform_cores from verify import find_undeclared_files profiles = ( emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir) ) # Detect destination conventions for core extras extras_prefix = _detect_extras_prefix(config, base_dest) is_slug_based, sys_to_slug = _detect_slug_structure(config) platform_systems = set(config.get("systems", {}).keys()) norm_map: dict[str, str] = {} if is_slug_based: for sid in platform_systems: norm_map[_norm_system_id(sid)] = sid # Use strict YAML names (no DB alias enrichment) so that files known # under an alias still get packed at the emulator's expected path. yaml_names: set[str] = set() for system in config.get("systems", {}).values(): for fe in system.get("files", []): name = fe.get("name", "") if name: yaml_names.add(name) undeclared = find_undeclared_files( config, emulators_dir, db, emu_profiles, target_cores=target_cores, include_all=include_all, declared_names=yaml_names, ) extras = [] seen_dests: set[str] = set(seen) for u in undeclared: if not u["in_repo"]: continue # For archive entries, use the archive name for resolution archive = u.get("archive") name = archive if archive else u["name"] raw_dest = archive if archive else (u.get("path") or u["name"]) # Directory path: append filename (e.g. "cafeLibs/" + "snd_user.rpl") dest = f"{raw_dest}{u['name']}" if raw_dest.endswith("/") else raw_dest # Slug-based platforms: prefix dest with system slug if is_slug_based: emu_name = u.get("profile") or u.get("emulator", "") profile = profiles.get(emu_name, {}) # Try finding profile by display name if key lookup failed if not profile: for pn, pp in profiles.items(): if pp.get("emulator") == emu_name: profile = pp break slug = _map_emulator_to_slug( profile, platform_systems, norm_map, sys_to_slug, ) if not slug: continue # can't place without slug dest = f"{slug}/{dest}" full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest if full_dest in seen_dests: continue seen_dests.add(full_dest) extra = { "name": name, "destination": dest, "required": u.get("required", False), "hle_fallback": u.get("hle_fallback", False), "source_emulator": u.get("emulator", ""), "source_profile": u.get("profile", ""), "source_system": u.get("system"), "source_systems": u.get("systems", []), "region": u.get("region"), "variant_group": u.get("variant_group"), } # Keep every reproducible identity constraint carried by the source # profile. Some profiles expose CRC32/SHA-256 only through their # validation block; dropping those fields here would let a same-named # but different payload enter a generated pack. expected = u.get("expected") or {} for identity in ("sha1", "md5", "sha256", "crc32", "size"): declared = u.get(identity) if declared in (None, "", []): declared = expected.get(identity) if declared not in (None, "", []): extra[identity] = declared extras.append(extra) # Second pass: find alternative destinations for files already in the pack. # A file declared by the platform or emitted above may also be needed at a # different path by another core (e.g. neocd/ vs root, same_cdi/bios/ vs root). # Only adds a copy when the file is ALREADY covered at a different path - # never introduces a file that wasn't selected by the first pass. # # Skip for slug-based platforms (RomM): alternative paths don't map to # the required {platform_slug}/{file} structure. if is_slug_based: return extras relevant = resolve_platform_cores(config, profiles, target_cores=target_cores) standalone_set = {str(c) for c in config.get("standalone_cores", [])} by_name = db.get("indexes", {}).get("by_name", {}) by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {}) # Build set of filenames already covered (platform baseline + first pass extras) # Enriched with canonical names from DB via MD5 (handles platform renaming) covered_names = expand_platform_declared_names(config, db) for e in extras: covered_names.add(e["name"]) for emu_name, profile in sorted(profiles.items()): if profile.get("type") in ("launcher", "alias"): continue if emu_name not in relevant: continue is_standalone = emu_name in standalone_set or bool( standalone_set & {str(c) for c in profile.get("cores", [])} ) for f in profile.get("files", []): fname = f.get("name", "") if not fname: continue # Only duplicate files already covered at another destination if fname not in covered_names: continue file_mode = f.get("mode") if file_mode == "standalone" and not is_standalone: continue if file_mode == "libretro" and is_standalone: continue # Skip files loaded from non-system directories (save_dir, content_dir) load_from = f.get("load_from", "") if load_from and load_from != "system_dir": continue if is_standalone: raw = f.get("standalone_path") or f.get("path") or fname else: raw = f.get("path") or fname dest = f"{raw}{fname}" if raw.endswith("/") else raw if dest == fname: continue # no alternative destination full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest if full_dest in seen_dests: continue # Check file exists in repo or data dirs if not ( by_name.get(fname) or by_name.get(dest.rsplit("/", 1)[-1]) or by_path_suffix.get(dest) ): continue seen_dests.add(full_dest) extras.append( { "name": fname, "destination": dest, "required": f.get("required", False), "hle_fallback": f.get("hle_fallback", False), "source_emulator": profile.get("emulator", emu_name), "source_profile": emu_name, "source_system": f.get("system"), "source_systems": list(profile.get("systems", [])), "region": f.get("region"), "variant_group": f.get("variant_group"), } ) # Archive prefix pass: cores that store BIOS archives in a subdirectory # (e.g. system/fbneo/neogeo.zip). When the archive is already covered at # the root, add a copy at the prefixed path so the core's .info firmware # check finds it. for emu_name, profile in sorted(profiles.items()): if profile.get("type") in ("launcher", "alias"): continue if emu_name not in relevant: continue prefix = profile.get("archive_prefix", "") if not prefix: continue profile_archives: dict[str, dict] = {} for f in profile.get("files", []): archive = f.get("archive", "") if archive: profile_archives.setdefault(archive, f) for archive_name, archive_entry in sorted(profile_archives.items()): if archive_name not in covered_names: continue dest = f"{prefix}/{archive_name}" full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest if full_dest in seen_dests: continue if not by_name.get(archive_name): continue seen_dests.add(full_dest) extras.append( { "name": archive_name, "destination": dest, "required": True, "hle_fallback": False, "source_emulator": profile.get("emulator", emu_name), "source_profile": emu_name, "source_system": archive_entry.get("system"), "source_systems": list(profile.get("systems", [])), "region": archive_entry.get("region"), "variant_group": archive_entry.get("variant_group"), } ) # Third pass: agnostic scan — for filename-agnostic cores, include all # DB files matching the system path prefix and size criteria. files_db = db.get("files", {}) for emu_name, profile in sorted(profiles.items()): if profile.get("type") in ("launcher", "alias"): continue if emu_name not in relevant: continue is_profile_agnostic = profile.get("bios_mode") == "agnostic" if not is_profile_agnostic: if not any(f.get("agnostic") for f in profile.get("files", [])): continue for f in profile.get("files", []): if not is_profile_agnostic and not f.get("agnostic"): continue fname = f.get("name", "") if not fname: continue # Derive path prefix from the representative file in the DB path_prefix = None sha1_list = by_name.get(fname, []) for sha1 in sha1_list: entry = files_db.get(sha1, {}) path = entry.get("path", "") if path: parts = path.rsplit("/", 1) if len(parts) == 2: path_prefix = parts[0] + "/" break if not path_prefix: # Fallback: try other files in the profile for the same system for other_f in profile.get("files", []): if other_f is f: continue other_name = other_f.get("name", "") for sha1 in by_name.get(other_name, []): entry = files_db.get(sha1, {}) path = entry.get("path", "") if path: parts = path.rsplit("/", 1) if len(parts) == 2: path_prefix = parts[0] + "/" break if path_prefix: break if not path_prefix: continue # Size criteria from the file entry min_size = f.get("min_size", 0) max_size = f.get("max_size", float("inf")) exact_size = f.get("size") if exact_size and not min_size: min_size = exact_size max_size = exact_size # Scan DB for all files under this prefix matching size for sha1, entry in files_db.items(): path = entry.get("path", "") if not path.startswith(path_prefix): continue size = entry.get("size", 0) if not (min_size <= size <= max_size): continue scan_name = entry.get("name", "") if not scan_name: continue dest = scan_name full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest if full_dest in seen_dests: continue seen_dests.add(full_dest) extras.append( { "name": scan_name, "destination": dest, "required": False, "hle_fallback": False, "source_emulator": profile.get("emulator", emu_name), "source_profile": emu_name, "source_system": f.get("system"), "source_systems": list(profile.get("systems", [])), "region": f.get("region"), "variant_group": f.get("variant_group"), "agnostic_scan": True, } ) return extras def _extra_system_ids(extra: dict) -> list[str]: """Return the narrowest system ownership preserved on a core extra.""" explicit = extra.get("source_system") if explicit: return [str(explicit)] return [str(value) for value in extra.get("source_systems", []) if value] def _emulator_region_group(emu_name: str, profile: dict, file_entry: dict) -> str: """Stable group ID for regional alternatives within an emulator profile.""" variant = file_entry.get("variant_group") if variant: return f"{emu_name}:variant:{variant}" system = file_entry.get("system") profile_systems = list(profile.get("systems", [])) if not system and len(profile_systems) == 1: system = profile_systems[0] return f"{emu_name}:system:{system or '_profile'}" def _build_readme( platform_name: str, platform_display: str, base_dest: str, total_files: int, num_systems: int, source: str = "full", contributors: list[dict] | None = None, regions: list[str] | None = None, fallback_systems: list[str] | None = None, one_per_slot: bool = False, undecidable_slots: list[str] | None = None, narrowings: list[tuple[str, str]] | None = None, system_filter: list[str] | None = None, ) -> str: """Build a personalized step-by-step README for each platform pack.""" narrowings = narrowings or [] sep = "=" * 50 header = ( f"{sep}\n" f" RETROBIOS - {platform_display} BIOS Pack\n" f" {total_files} files for {num_systems} systems\n" f"{sep}\n\n" ) guides: dict[str, str] = { "retroarch": ( "INSTALLATION GUIDE\n\n" " Option A: Automatic (recommended)\n" " ---------------------------------\n" " Run this in a terminal:\n\n" " curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh\n\n" " The script auto-detects your RetroArch install and copies\n" " files to the correct location.\n\n" " Option B: Manual (PC)\n" " ---------------------\n" " 1. Find your RetroArch system directory:\n" " - RetroArch > Settings > Directory > System/BIOS\n" " - Default: retroarch/system/\n" " 2. Extract all files from this archive directly into your system directory\n" " 3. Overwrite if asked\n\n" " Option C: Manual (handheld / SD card)\n" " -------------------------------------\n" " Anbernic, Retroid, Miyoo, Trimui, etc.:\n" " 1. Connect your SD card to your PC\n" " 2. Find the BIOS folder (usually BIOS/ or system/)\n" " 3. Extract all files from this archive directly into that folder\n" " 4. Eject SD card and reboot your device\n\n" " Common paths by device:\n" " Anbernic (ArkOS/JELOS): BIOS/\n" " Retroid (RetroArch): RetroArch/system/\n" " Miyoo Mini (Onion OS): BIOS/\n" " Steam Deck (RetroArch): ~/.config/retroarch/system/\n\n" ), "batocera": ( "INSTALLATION GUIDE\n\n" " Option A: Automatic (recommended)\n" " ---------------------------------\n" " Open a terminal (F1 from Batocera menu) and run:\n\n" " curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh\n\n" " Option B: Manual (network share)\n" " --------------------------------\n" " 1. On your PC, open the Batocera network share:\n" " - Windows: \\\\BATOCERA\\share\\bios\\\n" " - Mac/Linux: smb://batocera/share/bios/\n" " 2. Extract all files from this archive directly into the share\n" " 3. Overwrite if asked\n\n" " Option C: Manual (SD card)\n" " --------------------------\n" " 1. Put the SD card in your PC\n" " 2. Navigate to /userdata/bios/ on the SHARE partition\n" " 3. Extract all files from this archive directly into that folder\n\n" " NOTE: Dreamcast flash memory is named dc_nvmem.bin\n" " (if your setup asks for dc_flash.bin, same file).\n\n" ), "recalbox": ( "INSTALLATION GUIDE\n\n" " Option A: Automatic\n" " -------------------\n" " curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh\n\n" " Option B: Manual (network share)\n" " --------------------------------\n" " 1. On your PC, open the Recalbox network share:\n" " - Windows: \\\\RECALBOX\\share\\bios\\\n" " - Mac/Linux: smb://recalbox/share/bios/\n" " 2. Extract all files from this archive directly into the share\n\n" " Option C: Manual (SD card)\n" " --------------------------\n" " 1. Put the SD card in your PC\n" " 2. Navigate to /recalbox/share/bios/\n" " 3. Extract all files from this archive directly into that folder\n\n" ), "emudeck": ( "INSTALLATION GUIDE (Steam Deck / Linux)\n\n" " Option A: Automatic (recommended)\n" " ---------------------------------\n" " Open Konsole (or any terminal) and run:\n\n" " curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh\n\n" " The script places BIOS files AND sets up standalone\n" " emulator keys automatically.\n\n" " Option B: Manual\n" " ----------------\n" " 1. Open Dolphin file manager\n" " 2. Navigate to ~/Emulation/bios/\n" " 3. Extract all files from this archive directly into ~/Emulation/bios/\n\n" " STANDALONE EMULATORS (extra step)\n" " Switch and 3DS emulators need keys in specific folders:\n" " prod.keys -> ~/.local/share/yuzu/keys/\n" " prod.keys -> ~/.local/share/eden/keys/\n" " prod.keys -> ~/.config/Ryujinx/system/\n" " aes_keys.txt -> ~/Emulation/bios/citra/keys/\n" " The automatic installer handles this for you.\n\n" ), "retrodeck": ( "INSTALLATION GUIDE (Steam Deck / Linux)\n\n" " Option A: Automatic (recommended)\n" " ---------------------------------\n" " Open Konsole (or any terminal) and run:\n\n" " curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh\n\n" " Option B: Manual\n" " ----------------\n" " 1. Open Dolphin file manager\n" " 2. Show hidden files (Ctrl+H)\n" " 3. Navigate to ~/retrodeck/bios/\n" " 4. Extract all files from this archive directly into ~/retrodeck/bios/\n\n" " NOTE: RetroDECK uses its own BIOS checker. After\n" " copying, open RetroDECK > Tools > BIOS Checker to\n" " verify everything is detected.\n\n" ), "retrobat": ( "INSTALLATION GUIDE (Windows)\n\n" " Option A: Automatic (recommended)\n" " ---------------------------------\n" " Download and run install.bat from:\n" " https://github.com/Abdess/retrobios/releases\n\n" " Option B: Manual\n" " ----------------\n" " 1. Open your RetroBat installation folder\n" " 2. Navigate to the bios\\ subfolder\n" " (default: C:\\RetroBat\\bios\\)\n" " 3. Extract all files from this archive directly into your bios\\ folder\n" " 4. Overwrite if asked\n\n" ), "bizhawk": ( "INSTALLATION GUIDE\n\n" " 1. Open your BizHawk installation folder\n" " 2. Navigate to the Firmware subfolder:\n" " - Windows: BizHawk\\Firmware\\\n" " - Linux: ~/.config/BizHawk/Firmware/\n" " 3. Extract all files from this archive directly into your Firmware folder\n" " 4. In BizHawk: Config > Paths > Firmware should\n" " point to this folder\n\n" ), "romm": ( "INSTALLATION GUIDE (RomM server)\n\n" " 1. Locate your RomM library folder\n" " 2. Navigate to the bios/ subdirectory\n" " 3. Extract all files from this archive directly into that folder\n" " 4. Restart the RomM service to detect new files\n\n" ), "retropie": ( "INSTALLATION GUIDE (Raspberry Pi)\n\n" " Option A: Via network share\n" " --------------------------\n" " 1. On your PC, open: \\\\RETROPIE\\bios\\\n" " 2. Extract all files from this archive directly into that folder\n\n" " Option B: Via SSH\n" " -----------------\n" " 1. SSH into your Pi: ssh pi@retropie\n" " 2. Copy files to ~/RetroPie/BIOS/\n\n" " Option C: Via SD card\n" " ---------------------\n" " 1. Put the SD card in your PC\n" " 2. Navigate to /home/pi/RetroPie/BIOS/\n" " 3. Extract all files from this archive directly into that folder\n\n" ), } # Lakka uses same guide as RetroArch guides["lakka"] = guides["retroarch"] guide = guides.get( platform_name, ( f"INSTALLATION\n\n" f" 1. Extract all files from this archive directly into your BIOS directory\n" f" 2. Overwrite if asked\n\n" ), ) if regions: region_help = ( " - Wrong region? This pack was filtered. Only the\n" " best-matching BIOS was kept per system. Use the\n" " unfiltered pack to play imports.\n" ) else: region_help = ( " - Wrong region? Some systems have regional BIOS\n" " variants (USA/EUR/JAP). All are included.\n" ) footer = ( "TROUBLESHOOTING\n\n" " - Core says BIOS missing? Check the exact filename\n" " and make sure it's in the right subfolder.\n" f"{region_help}" " - Need help? https://github.com/Abdess/retrobios/issues\n\n" f"{sep}\n" f" https://github.com/Abdess/retrobios\n" f"{sep}\n" ) source_info = "" if source == "platform": source_info = ( "PACK TYPE: Platform Only\n\n" f" This pack contains only files declared by {platform_display}.\n" " Core extras from emulator profiles are not included.\n" " Use the Full pack for maximum coverage.\n\n" ) elif source == "truth": source_info = ( "PACK TYPE: Ground Truth\n\n" " This pack contains files that emulators actually load,\n" " based on source code analysis of emulator profiles.\n" " Independent of platform scraper accuracy.\n\n" ) region_info = "" if regions: pretty = ", ".join( " ".join(w.title() for w in slug.split("-")) for slug in regions ) region_info = ( "PACK TYPE: Region Filtered\n\n" f" Region priority: {pretty}\n\n" " Only the best-matching BIOS was kept for each system.\n" ) if fallback_systems: listed = "\n".join(f" {s}" for s in fallback_systems) region_info += ( "\n These systems have no BIOS in those regions, so all\n" " of theirs were kept:\n" f"{listed}\n" ) region_info += ( "\n This shrinks the pack. It does not change how cores pick\n" " a BIOS: most already select per region from fixed filename\n" " lists driven by the game's region. Loading imports from\n" " another region may need the unfiltered pack.\n\n" ) slot_info = "" if one_per_slot: slot_info = ( " Where a core declares which BIOS it prefers, only that one was\n" " kept for each system and region. Systems whose cores declare no\n" " order keep all of theirs.\n" ) if undecidable_slots: slot_info += ( f" {len(undecidable_slots)} slot(s) had no declared order.\n" ) slot_info += "\n" narrowed = "" labels = [label for _tag, label in narrowings] if system_filter: labels.append(f"systems {', '.join(system_filter)}") if labels: listed = "".join(f" {label}\n" for label in labels) narrowed = ( "PACK TYPE: Narrowed\n\n" " This pack holds fewer files than the full one:\n" f"{listed}" "\n The unfiltered pack is the one to use when in doubt.\n\n" ) credits = "" if contributors: credits = "\nCONTRIBUTORS\n\n" for cb in contributors: username = cb.get("username", "") credits += f" @{username}\n" credits += "\n" return ( header + narrowed + source_info + region_info + slot_info + guide + credits + footer ) def _build_agnostic_rename_readme( destination: str, original: str, alternatives: list[str], ) -> str: """Build a README explaining an agnostic file rename.""" lines = [ "This file was renamed for compatibility:", f" {destination} <- {original}", "", ] if alternatives: lines.append("All variants included in this pack:") for alt in sorted(alternatives): lines.append(f" {alt}") lines.append("") lines.append(f"To use a different variant, rename it to: {destination}") return "\n".join(lines) + "\n" def generate_pack( platform_name: str, platforms_dir: str, db: dict, bios_dir: str, output_dir: str, include_extras: bool = False, emulators_dir: str = "emulators", zip_contents: dict | None = None, data_registry: dict | None = None, emu_profiles: dict | None = None, target_cores: set[str] | None = None, required_only: bool = False, system_filter: list[str] | None = None, precomputed_extras: list[dict] | None = None, source: str = "full", flatten: bool = True, regions: list[str] | None = None, target_name: str | None = None, one_per_slot: bool = False, offline: bool | None = None, ) -> str | None: """Generate a ZIP pack for a platform. Returns the path to the generated ZIP, or None on failure. """ config = load_platform_config(platform_name, platforms_dir) if zip_contents is None: zip_contents = {} verification_mode = config.get("verification_mode", "existence") platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") version = config.get("version", config.get("dat_version", "")) version_tag = f"_{version.replace(' ', '')}" if version else "" narrowings = _narrowings( source, regions, target_name, one_per_slot, required_only ) narrow_tags = "".join(tag for tag, _label in narrowings) sys_tag = "" if system_filter: display_parts = [] for sid in system_filter: s = sid.lower().replace("_", "-") for prefix in MANUFACTURER_PREFIXES: if s.startswith(prefix): s = s[len(prefix) :] break parts = s.split("-") display_parts.append("_".join(p.title() for p in parts if p)) sys_tag = "_" + "_".join(display_parts) zip_name = f"{platform_display.replace(' ', '_')}{version_tag}{narrow_tags}_BIOS_Pack{sys_tag}.zip" zip_path = os.path.join(output_dir, zip_name) os.makedirs(output_dir, exist_ok=True) # Case-insensitive dedup only for platforms targeting Windows/macOS. # Linux-only platforms (Batocera, Recalbox, RetroDECK, Lakka, RomM) # are case-sensitive and may have distinct files like DISK.ROM vs disk.rom. case_insensitive = config.get("case_insensitive_fs", False) total_files = 0 missing_files = [] # Core extras whose local copy contradicts the profile hash: packed, # reported, and never counted against the platform's own file total. core_discrepancies: list[str] = [] user_provided = [] seen_destinations: set[str] = set() seen_lower: set[str] = set() # only used when case_insensitive=True seen_parents: set[str] = ( set() ) # parent dirs of added files (path conflict detection) # Per-destination status. ``excluded`` means a same-named local payload # exists but contradicts the declaration and is intentionally not shipped. # It is distinct from a genuine coverage gap (``missing``). file_status: dict[str, str] = {} file_reasons: dict[str, str] = {} # Build emulator-level validation index (same as verify.py) validation_index = {} if emu_profiles: validation_index = _build_validation_index(emu_profiles) # Filter systems by target if specified from common import resolve_platform_cores plat_cores = ( resolve_platform_cores(config, emu_profiles or {}) if target_cores else None ) pack_systems = filter_systems_by_target( config.get("systems", {}), emu_profiles or {}, target_cores, platform_cores=plat_cores, ) if system_filter: from common import _norm_system_id norm_filter = {_norm_system_id(s) for s in system_filter} filtered = { sid: sys_data for sid, sys_data in pack_systems.items() if sid in system_filter or _norm_system_id(sid) in norm_filter } if not filtered: available = sorted(pack_systems.keys())[:10] print( f" WARNING: no systems matched filter {system_filter} " f"(available: {', '.join(available)})" ) return None pack_systems = filtered # Constrained-entry preference: platforms may declare the same # destination several times (bare in one system, hash-constrained in # another). First-come dedup would let the bare entry pack a wrong # same-named file, so when a hash-constrained sibling resolves to a # matching repo file, it claims the destination instead. preferred_entries: dict[str, int] = {} if source != "truth": dest_entries: dict[str, list[dict]] = {} for sys_id, system in sorted(pack_systems.items()): for file_entry in system.get("files", []): if required_only and file_entry.get("required") is False: continue dest = _sanitize_path( file_entry.get("destination", file_entry.get("name", "")) ) if not dest: continue full = f"{base_dest}/{dest}" if base_dest else dest dest_entries.setdefault(full, []).append(file_entry) for full, entries in dest_entries.items(): if len(entries) < 2: continue constrained = [ fe for fe in entries if fe.get("md5") or fe.get("sha1") or fe.get("zipped_file") ] if not constrained: continue best = None for fe in constrained: _lp, _st = resolve_file( fe, db, bios_dir, zip_contents, data_dir_registry=data_registry, offline=offline, ) if _lp and _st == "md5_exact": best = fe break if best is None and _lp and resolution_is_hash_exact(_st): best = fe if best is not None: preferred_entries[full] = id(best) # Region selection is decided once, over both the platform baseline and the # core extras, so the two phases below consult a single set. Grouping uses a # dedicated extras pass with an empty seen set: it only needs group # membership, never destinations, and a superset of members is the safe # direction. Runs only when --region is given. region_drops: set[str] = set() region_fallbacks: list[str] = [] if regions: region_index = region_mod.build_region_index(emu_profiles or {}) region_groups: dict[str, list[tuple[str, str]]] = {} for sys_id, system in pack_systems.items(): members = region_groups.setdefault(sys_id, []) for file_entry in system.get("files", []): dest = _sanitize_path( file_entry.get("destination", file_entry.get("name", "")) ) if dest: members.append((dest, file_entry.get("name", ""))) if source != "platform": for fe in _collect_emulator_extras( config, emulators_dir, db, set(), base_dest, emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ): dest = _sanitize_path(fe.get("destination", fe.get("name", ""))) if not dest: continue systems = _extra_system_ids(fe) or ["_extras"] for sys_id in systems: variant = fe.get("variant_group") group_id = f"{sys_id}:variant:{variant}" if variant else sys_id region_groups.setdefault(group_id, []).append( (dest, fe.get("name", "")) ) region_drops = region_mod.resolve_region_drops( region_groups, region_index, regions ) region_fallbacks = region_mod.fallback_groups( region_groups, region_index, regions ) slot_undecidable: list[str] = [] if one_per_slot: members_by_system = _pack_member_groups( config, pack_systems, emulators_dir, db, base_dest, emu_profiles, target_cores, source, ) slot_groups = { sys_id: [(d, n) for d, n in members if d not in region_drops] for sys_id, members in members_by_system.items() } slot_drops, slot_undecidable = slot_mod.resolve_slot_drops( slot_groups, slot_mod.build_slot_index(emu_profiles or {}) ) region_drops = region_drops | slot_drops if slot_undecidable: print( f" {len(slot_undecidable)} slot(s) with no declared order: " f"every candidate kept" ) with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: if source != "truth": for sys_id, system in sorted(pack_systems.items()): for file_entry in system.get("files", []): if required_only and file_entry.get("required") is False: continue dest = _sanitize_path(file_entry.get("destination", file_entry["name"])) if region_drops and dest in region_drops: continue if not dest: # EmuDeck-style entries (system:md5 whitelist, no filename). fkey = f"{sys_id}/{file_entry.get('name', '')}" md5 = file_entry.get("md5", "") if md5 and md5 in db.get("indexes", {}).get("by_md5", {}): file_status.setdefault(fkey, "ok") else: file_status[fkey] = "missing" continue if base_dest: full_dest = f"{base_dest}/{dest}" else: full_dest = dest dedup_key = full_dest already_packed = dedup_key in seen_destinations or ( case_insensitive and dedup_key.lower() in seen_lower ) preferred = preferred_entries.get(dedup_key) if preferred is not None and id(file_entry) != preferred: continue if _has_path_conflict(full_dest, seen_destinations, seen_parents): # Content ships under the conflicting shape (file vs dir, # e.g. SGB1.sfc): count it so pack totals match verify.py, # which resolves each declaration independently file_status.setdefault(dedup_key, "ok") continue storage = file_entry.get("storage", "embedded") if storage == "user_provided": if already_packed: continue seen_destinations.add(dedup_key) _register_path(dedup_key, seen_destinations, seen_parents) if case_insensitive: seen_lower.add(dedup_key.lower()) file_status.setdefault(dedup_key, "ok") instructions = file_entry.get( "instructions", "Please provide this file manually." ) instr_name = f"INSTRUCTIONS_{file_entry['name']}.txt" instr_path = ( f"{base_dest}/{instr_name}" if base_dest else instr_name ) _write_generated_member( zf, _flat(instr_path, base_dest, flatten), f"File needed: {file_entry['name']}\n\n{instructions}\n", ) user_provided.append(file_entry["name"]) total_files += 1 continue local_path, status = resolve_file( file_entry, db, bios_dir, zip_contents, data_dir_registry=data_registry, offline=offline, ) if status == "external": file_ext = os.path.splitext(file_entry["name"])[1] or "" with tempfile.NamedTemporaryFile( delete=False, suffix=file_ext ) as tmp: tmp_path = tmp.name try: if download_external(file_entry, tmp_path): extract = file_entry.get("extract", False) if extract and tmp_path.endswith(".zip"): _extract_zip_to_archive(tmp_path, _flat(full_dest, base_dest, flatten), zf) else: zf.write(tmp_path, _flat(full_dest, base_dest, flatten)) seen_destinations.add(dedup_key) _register_path(dedup_key, seen_destinations, seen_parents) if case_insensitive: seen_lower.add(dedup_key.lower()) file_status.setdefault(dedup_key, "ok") total_files += 1 else: file_status[dedup_key] = "missing" file_reasons[dedup_key] = "external download failed" finally: if os.path.exists(tmp_path): os.unlink(tmp_path) continue if status == "not_found": # Agnostic fallback: if an agnostic core covers this system, # find any matching file in the DB by_name = db.get("indexes", {}).get("by_name", {}) files_db = db.get("files", {}) agnostic_path = None agnostic_resolved = False if emu_profiles: for _emu_key, _emu_prof in emu_profiles.items(): if _emu_prof.get("bios_mode") != "agnostic": continue if sys_id not in set(_emu_prof.get("systems", [])): continue for _ef in _emu_prof.get("files", []): ef_name = _ef.get("name", "") for _sha1 in by_name.get(ef_name, []): _entry = files_db.get(_sha1, {}) _path = _entry.get("path", "") if _path: _prefix = _path.rsplit("/", 1)[0] + "/" _min = _ef.get("min_size", 0) _max = _ef.get("max_size", float("inf")) if _ef.get("size") and not _min: _min = _ef["size"] _max = _ef["size"] for _s, _e in files_db.items(): if _e.get("path", "").startswith(_prefix): if _min <= _e.get("size", 0) <= _max: if os.path.exists(_e["path"]): local_path = _e["path"] agnostic_path = _prefix agnostic_resolved = True break break if agnostic_resolved: break if agnostic_resolved: break if agnostic_resolved and local_path: # Write rename README original_name = os.path.basename(local_path) dest_name = file_entry.get("name", "") if original_name != dest_name and agnostic_path: alt_names = [] for _s, _e in files_db.items(): _p = _e.get("path", "") if _p.startswith(agnostic_path): _n = _e.get("name", "") if _n and _n != original_name: alt_names.append(_n) readme_text = _build_agnostic_rename_readme( dest_name, original_name, alt_names, ) readme_name = f"RENAMED_{dest_name}.txt" readme_full = ( f"{base_dest}/{readme_name}" if base_dest else readme_name ) if readme_full not in seen_destinations: _write_generated_member( zf, _flat(readme_full, base_dest, flatten), readme_text, ) seen_destinations.add(readme_full) status = "agnostic_fallback" # Fall through to normal packing below else: if not already_packed: file_status[dedup_key] = "missing" file_reasons[dedup_key] = "not found" continue if status == "hash_mismatch" and verification_mode != "existence": zf_name = file_entry.get("zipped_file") if zf_name and local_path: inner_md5_raw = file_entry.get("md5", "") inner_md5_list = ( [m.strip() for m in inner_md5_raw.split(",") if m.strip()] if inner_md5_raw else [""] ) zip_ok = False last_result = "not_in_zip" for md5_candidate in inner_md5_list: last_result = check_inside_zip( local_path, zf_name, md5_candidate ) if last_result == "ok": zip_ok = True break if zip_ok: status = "zip_exact" file_status.setdefault(dedup_key, "ok") elif last_result == "not_in_zip": file_status[dedup_key] = "excluded" file_reasons[dedup_key] = f"{zf_name} not found inside ZIP" elif last_result == "error": file_status[dedup_key] = "excluded" file_reasons[dedup_key] = "cannot read ZIP" else: file_status[dedup_key] = "excluded" file_reasons[dedup_key] = ( f"{zf_name} MD5 mismatch inside ZIP" ) if not zip_ok: continue else: file_status[dedup_key] = "excluded" file_reasons[dedup_key] = "hash mismatch" continue else: # Existence platforms accept any file at the declared path: # their code never reads the bytes. An upstream hash that # contradicts a local dump is reported, not acted on. if status == "hash_mismatch" and local_path: declared = file_entry.get("md5", "") or file_entry.get( "sha1", "" ) actual = compute_hashes(local_path) file_reasons.setdefault( dedup_key, f"packed per {platform_display} existence check; " f"declared hash {declared}, file md5 " f"{actual['md5']} sha1 {actual['sha1']}", ) file_status.setdefault(dedup_key, "ok") # Emulator-level validation: informational only for platform packs. # Platform verification (existence/md5) is the authority for pack status. # Emulator checks are supplementary -logged but don't downgrade. # When a discrepancy is found, try to find a file satisfying both. if ( file_status.get(dedup_key) == "ok" and local_path and validation_index ): fname = file_entry.get("name", "") check = check_file_validation( local_path, fname, validation_index, bios_dir ) if check: reason, emus_list = check better = _find_candidate_satisfying_both( file_entry, db, local_path, validation_index, bios_dir, ) if better: local_path = better else: emus = ", ".join(emus_list) file_reasons.setdefault( dedup_key, f"{platform_display} says OK but {emus} says {reason}", ) if already_packed: continue seen_destinations.add(dedup_key) _register_path(dedup_key, seen_destinations, seen_parents) if case_insensitive: seen_lower.add(dedup_key.lower()) extract = file_entry.get("extract", False) flat_dest = _flat(full_dest, base_dest, flatten) if extract and local_path.endswith(".zip"): _extract_zip_to_archive(local_path, flat_dest, zf) elif local_path.endswith(".zip"): _add_zip_to_pack(local_path, flat_dest, zf, file_entry) else: zf.write(local_path, flat_dest) total_files += 1 # Core requirements: files platform's cores need but YAML doesn't declare if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) if source == "platform": core_files = [] elif precomputed_extras is not None: core_files = precomputed_extras elif system_filter and source != "truth": core_files = [] else: core_files = _collect_emulator_extras( config, emulators_dir, db, seen_destinations, base_dest, emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ) # Truth mode + system_filter: filter core files by system ID if system_filter and source == "truth" and core_files: from common import _norm_system_id norm_filter = {_norm_system_id(s) for s in system_filter} | set( system_filter ) core_files = [ fe for fe in core_files if ( set(_extra_system_ids(fe)) | {_norm_system_id(s) for s in _extra_system_ids(fe)} ) & norm_filter ] core_count = 0 for fe in core_files: if required_only and fe.get("required") is False: continue dest = _sanitize_path(fe.get("destination", fe["name"])) if region_drops and dest in region_drops: continue if not dest: continue # Core extras: _collect_emulator_extras already adjusted # destinations for slug-based platforms. Apply the effective # prefix (base_dest, or inferred from YAML when base_dest is # empty — e.g. RetroDECK infers "bios"). extras_pfx = _detect_extras_prefix(config, base_dest) if extras_pfx: if not dest.startswith(f"{extras_pfx}/"): full_dest = f"{extras_pfx}/{dest}" else: full_dest = dest else: full_dest = dest if full_dest in seen_destinations: continue # Skip case-insensitive duplicates (Windows/macOS FS safety) if full_dest.lower() in seen_lower and case_insensitive: continue # Skip file/directory path conflicts (e.g., SGB1.sfc file vs SGB1.sfc/ dir) if _has_path_conflict(full_dest, seen_destinations, seen_parents): continue dest_hint = fe.get("destination", "") local_path, status = resolve_file( fe, db, bios_dir, zip_contents, dest_hint=dest_hint, data_dir_registry=data_registry, offline=offline, ) if status in ("not_found", "external", "user_provided") or not local_path: continue if status == "hash_mismatch": # The core's declared hash comes from its source, the local dump # is what the collection holds. Shipping it keeps the emulator # working when the code never reads the hash; the divergence is # reported so it can be resolved at the profile or the dump. # file_status tracks platform declarations only, so the report # goes to its own list and never moves the pack's own count. core_discrepancies.append( f"{full_dest} -declared hash of " f"{fe.get('source_profile') or fe.get('source_emulator', 'core profile')}" " does not match the packed copy" ) flat_dest = _flat(full_dest, base_dest, flatten) if local_path.endswith(".zip"): _add_zip_to_pack(local_path, flat_dest, zf, fe) else: zf.write(local_path, flat_dest) if file_status.get(full_dest) in ("missing", "excluded"): previous = file_status[full_dest] file_status[full_dest] = "ok" if previous == "excluded": source_name = fe.get("source_profile") or fe.get( "source_emulator", "core profile" ) file_reasons[full_dest] = ( "platform-declared hash unavailable; " f"packed the validated {source_name} requirement" ) else: file_reasons.pop(full_dest, None) seen_destinations.add(full_dest) _register_path(full_dest, seen_destinations, seen_parents) if case_insensitive: seen_lower.add(full_dest.lower()) core_count += 1 total_files += 1 # Data directories from _data_dirs.yml for sys_id, system in sorted(pack_systems.items()): for dd in system.get("data_directories", []): ref_key = dd.get("ref", "") if not ref_key or not data_registry or ref_key not in data_registry: continue entry = data_registry[ref_key] allowed = entry.get("for_platforms") if allowed and platform_name not in allowed: continue local_path = entry.get("local_cache", "") if not local_path or not os.path.isdir(local_path): print( f" WARNING: data directory '{ref_key}' not cached at {local_path} -run refresh_data_dirs.py" ) continue dd_dest = dd.get("destination", "") if base_dest and dd_dest: dd_prefix = f"{base_dest}/{dd_dest}" elif base_dest: dd_prefix = base_dest else: dd_prefix = dd_dest for root, _dirs, filenames in os.walk(local_path): for fname in filenames: src = os.path.join(root, fname) rel = os.path.relpath(src, local_path) full = f"{dd_prefix}/{rel}" if full in seen_destinations or ( full.lower() in seen_lower and case_insensitive ): continue if _has_path_conflict(full, seen_destinations, seen_parents): continue seen_destinations.add(full) _register_path(full, seen_destinations, seen_parents) if case_insensitive: seen_lower.add(full.lower()) zf.write(src, _flat(full, base_dest, flatten)) total_files += 1 # README.txt for users -personalized step-by-step per platform num_systems = len(pack_systems) _registry_path = Path(platforms_dir) / "_registry.yml" _pack_registry: dict = {} if _registry_path.exists(): with open(_registry_path) as _rf: _pack_registry = (yaml_load(_rf) or {}).get("platforms", {}) readme_text = _build_readme( platform_name, platform_display, base_dest, total_files, num_systems, source=source, contributors=_pack_registry.get(platform_name, {}).get("contributed_by", []), regions=regions, fallback_systems=region_fallbacks, one_per_slot=one_per_slot, undecidable_slots=slot_undecidable, narrowings=narrowings, system_filter=system_filter, ) _write_generated_member(zf, "README.txt", readme_text) files_ok = sum(1 for s in file_status.values() if s == "ok") files_untested = sum(1 for s in file_status.values() if s == "untested") files_excluded = sum(1 for s in file_status.values() if s == "excluded") files_miss = sum(1 for s in file_status.values() if s == "missing") total_checked = len(file_status) parts = [f"{files_ok}/{total_checked} files OK"] if files_untested: parts.append(f"{files_untested} untested") if files_excluded: parts.append(f"{files_excluded} unsafe excluded") if files_miss: parts.append(f"{files_miss} missing") if source == "platform": print( f" {zip_path}: {total_files} files packed (platform baseline only), " f"{', '.join(parts)} [{verification_mode}]" ) elif source == "truth": print( f" {zip_path}: {total_files} files packed (ground truth only), " f"{core_count} from emulator profiles [{verification_mode}]" ) else: baseline = total_files - core_count print( f" {zip_path}: {total_files} files packed ({baseline} baseline + " f"{core_count} from cores), {', '.join(parts)} [{verification_mode}]" ) for key, reason in sorted(file_reasons.items()): status = file_status.get(key, "") if status == "untested": label = "UNTESTED" elif status == "excluded": label = "EXCLUDED" elif status == "missing": label = "MISSING" else: label = "DISCREPANCY" print(f" {label}: {key} -{reason}") for name in missing_files: print(f" MISSING: {name}") for note in sorted(core_discrepancies): print(f" DISCREPANCY: {note}") return zip_path def _extract_zip_to_archive( source_zip: str, dest_prefix: str, target_zf: zipfile.ZipFile ): """Extract contents of a source ZIP into target ZIP under dest_prefix.""" with zipfile.ZipFile(source_zip, "r") as src: for info in src.infolist(): if info.is_dir(): continue clean_name = _sanitize_path(info.filename) if not clean_name: continue data = src.read(info.filename) target_path = f"{dest_prefix}/{clean_name}" if dest_prefix else clean_name target_zf.writestr(target_path, data) def _add_zip_to_pack( source_zip: str, dest_path: str, target_zf: zipfile.ZipFile, file_entry: dict | None = None, ): """Write a source ZIP into the pack. A declared outer hash (md5/sha1/crc32) means the platform or profile verifies the ZIP bytes themselves (Batocera md5sum, System.dat), so the resolved file is copied unchanged: a rebuild would change the outer hash and fail the native check. Entries with zipped_file declare the hash of a ROM inside the ZIP (checkInsideZip), which a content-preserving rebuild does not affect. """ if ( file_entry and not file_entry.get("zipped_file") and ( file_entry.get("md5") or file_entry.get("sha1") or file_entry.get("crc32") ) ): target_zf.write(source_zip, dest_path) return _normalize_zip_for_pack(source_zip, dest_path, target_zf) def _normalize_zip_for_pack( source_zip: str, dest_path: str, target_zf: zipfile.ZipFile ): """Add a MAME BIOS ZIP to the pack as a deterministic rebuild. Instead of copying the original ZIP (with non-deterministic metadata), extracts the ROM atoms, rebuilds the ZIP deterministically, and writes the normalized version into the pack. This ensures: - Same ROMs -> same ZIP hash in every pack build - No dependency on how the user built their MAME ROM set - Bit-identical ZIPs across platforms and build times """ import tempfile as _tmp # tmp/ is gitignored, so a fresh clone does not carry it. WSL keeps /tmp # on a 4 GB tmpfs that MAME sets overflow, hence the repo-local scratch. os.makedirs("tmp", exist_ok=True) tmp_fd, tmp_path = _tmp.mkstemp(suffix=".zip", dir="tmp") os.close(tmp_fd) try: rebuild_zip_deterministic(source_zip, tmp_path) target_zf.write(tmp_path, dest_path) except zipfile.BadZipFile: # Corrupt source ZIP: copy as-is (will be flagged by verify) target_zf.write(source_zip, dest_path) finally: os.unlink(tmp_path) # Emulator/system mode pack generation def _resolve_destination( file_entry: dict, pack_structure: dict | None, standalone: bool ) -> str: """Resolve the ZIP destination path for a file entry.""" # 1. standalone_path override if standalone and file_entry.get("standalone_path"): rel = file_entry["standalone_path"] # 2. path field elif file_entry.get("path"): rel = file_entry["path"] # 3. name fallback else: rel = file_entry.get("name", "") rel = _sanitize_path(rel) # Prepend pack_structure prefix if pack_structure: mode_key = "standalone" if standalone else "libretro" prefix = pack_structure.get(mode_key, "") if prefix: rel = f"{prefix}/{rel}" return rel def generate_emulator_pack( profile_names: list[str], emulators_dir: str, db: dict, bios_dir: str, output_dir: str, standalone: bool = False, zip_contents: dict | None = None, required_only: bool = False, regions: list[str] | None = None, offline: bool | None = None, ) -> str | None: """Generate a ZIP pack for specific emulator profiles.""" all_profiles = load_emulator_profiles(emulators_dir, skip_aliases=False) if zip_contents is None: zip_contents = build_zip_contents_index(db) # Resolve and validate profile names selected: list[tuple[str, dict]] = [] for name in profile_names: if name not in all_profiles: available = sorted( k for k, v in all_profiles.items() if v.get("type") not in ("alias", "test") ) print(f"Error: emulator '{name}' not found", file=sys.stderr) print(f"Available: {', '.join(available[:10])}...", file=sys.stderr) return None p = all_profiles[name] if p.get("type") == "alias": alias_of = p.get("alias_of", "?") print( f"Error: {name} is an alias of {alias_of} -use --emulator {alias_of}", file=sys.stderr, ) return None if p.get("type") == "launcher": print( f"Error: {name} is a launcher -use the emulator it launches", file=sys.stderr, ) return None ptype = p.get("type", "libretro") if standalone and "standalone" not in ptype: print( f"Error: {name} ({ptype}) does not support --standalone", file=sys.stderr, ) return None selected.append((name, p)) # ZIP naming display_names = [p.get("emulator", n).replace(" ", "") for n, p in selected] narrow_tags = "".join( tag for tag, _label in _narrowings( "full", regions, None, False, required_only, standalone ) ) zip_name = "_".join(display_names) + f"{narrow_tags}_BIOS_Pack.zip" zip_path = os.path.join(output_dir, zip_name) os.makedirs(output_dir, exist_ok=True) # One group per system (or explicit variant_group) inside each profile. # Multi-system cores such as O2EM therefore retain a fallback BIOS for # every system instead of letting one matching system empty another. region_drops: set[str] = set() if regions: region_index = region_mod.build_region_index(dict(selected)) region_groups: dict[str, list[tuple[str, str]]] = {} for emu_name, profile in sorted(selected): structure = profile.get("pack_structure") for fe in filter_files_by_mode(profile.get("files", []), standalone): dest = _resolve_destination(fe, structure, standalone) if dest: group_id = _emulator_region_group(emu_name, profile, fe) region_groups.setdefault(group_id, []).append( (dest, fe.get("name", "")) ) region_drops = region_mod.resolve_region_drops( region_groups, region_index, regions ) total_files = 0 missing_files = [] seen_destinations: set[str] = set() seen_lower: set[str] = set() seen_parents: set[str] = ( set() ) # parent dirs of added files (path conflict detection) seen_hashes: set[str] = set() # SHA1 dedup for same file, different path data_dir_notices: list[str] = [] data_registry = load_data_dir_registry( os.path.join(os.path.dirname(__file__), "..", "platforms") ) with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: for emu_name, profile in sorted(selected): pack_structure = profile.get("pack_structure") files = filter_files_by_mode(profile.get("files", []), standalone) for dd in profile.get("data_directories", []): ref_key = dd.get("ref", "") if not ref_key or not data_registry or ref_key not in data_registry: if ref_key: data_dir_notices.append(ref_key) continue entry = data_registry[ref_key] local_cache = entry.get("local_cache", "") if not local_cache or not os.path.isdir(local_cache): data_dir_notices.append(ref_key) continue dd_dest = dd.get("destination", "") if pack_structure: mode_key = "standalone" if standalone else "libretro" prefix = pack_structure.get(mode_key, "") if prefix: dd_dest = f"{prefix}/{dd_dest}" if dd_dest else prefix for root, _dirs, filenames in os.walk(local_cache): for fname in filenames: src = os.path.join(root, fname) rel = os.path.relpath(src, local_cache) full = f"{dd_dest}/{rel}" if dd_dest else rel if full.lower() in seen_lower: continue if _has_path_conflict(full, seen_destinations, seen_parents): continue seen_destinations.add(full) _register_path(full, seen_destinations, seen_parents) seen_lower.add(full.lower()) zf.write(src, full) total_files += 1 if not files: print(f" No files needed for {profile.get('emulator', emu_name)}") continue # Collect archives as atomic units archives: set[str] = set() for fe in files: archive = fe.get("archive") if archive: archives.add(archive) # Pack archives as units archive_prefix = profile.get("archive_prefix", "") for archive_name in sorted(archives): archive_dest = _sanitize_path(archive_name) if archive_prefix: archive_dest = f"{archive_prefix}/{archive_dest}" if pack_structure: mode_key = "standalone" if standalone else "libretro" prefix = pack_structure.get(mode_key, "") if prefix: archive_dest = f"{prefix}/{archive_dest}" if archive_dest.lower() in seen_lower: continue if _has_path_conflict(archive_dest, seen_destinations, seen_parents): continue # Prefer the profile's own entry for the archive (carries # hashes for exact resolution when several dumps share a name) archive_entry = next( ( f for f in files if f.get("name") == archive_name and not f.get("archive") ), {"name": archive_name}, ) local_path, status = resolve_file( archive_entry, db, bios_dir, zip_contents, data_dir_registry=data_registry, offline=offline, ) if local_path and status not in ("not_found",): if local_path.endswith(".zip"): _add_zip_to_pack(local_path, archive_dest, zf, archive_entry) else: zf.write(local_path, archive_dest) seen_destinations.add(archive_dest) _register_path(archive_dest, seen_destinations, seen_parents) seen_lower.add(archive_dest.lower()) total_files += 1 else: missing_files.append(archive_name) # Pack individual files (skip archived ones) for fe in files: if required_only and fe.get("required") is False: continue if fe.get("archive"): continue dest = _resolve_destination(fe, pack_structure, standalone) if not dest: continue if region_drops and dest in region_drops: continue if dest.lower() in seen_lower: continue if _has_path_conflict(dest, seen_destinations, seen_parents): continue storage = fe.get("storage", "embedded") if storage == "user_provided": seen_destinations.add(dest) _register_path(dest, seen_destinations, seen_parents) seen_lower.add(dest.lower()) instr = fe.get("instructions", "Please provide this file manually.") instr_name = f"INSTRUCTIONS_{fe['name']}.txt" _write_generated_member( zf, instr_name, f"File needed: {fe['name']}\n\n{instr}\n" ) total_files += 1 continue dest_hint = fe.get("path", "") local_path, status = resolve_file( fe, db, bios_dir, zip_contents, dest_hint=dest_hint, data_dir_registry=data_registry, offline=offline, ) if status == "external": file_ext = os.path.splitext(fe["name"])[1] or "" with tempfile.NamedTemporaryFile( delete=False, suffix=file_ext ) as tmp: tmp_path = tmp.name try: if download_external(fe, tmp_path): zf.write(tmp_path, dest) seen_destinations.add(dest) _register_path(dest, seen_destinations, seen_parents) seen_lower.add(dest.lower()) total_files += 1 else: missing_files.append(fe["name"]) finally: if os.path.exists(tmp_path): os.unlink(tmp_path) continue if status in ("not_found", "user_provided") or not local_path: missing_files.append(fe["name"]) continue # SHA1 dedup: skip if same physical file AND same destination # (but allow same file to be packed under different destinations, # e.g., IPL.bin in GC/USA/ and GC/EUR/ from same source) if local_path: real = os.path.realpath(local_path) dedup_key_hash = f"{real}:{dest}" if dedup_key_hash in seen_hashes: continue seen_hashes.add(dedup_key_hash) if local_path.endswith(".zip"): _add_zip_to_pack(local_path, dest, zf, fe) else: zf.write(local_path, dest) seen_destinations.add(dest) _register_path(dest, seen_destinations, seen_parents) seen_lower.add(dest.lower()) total_files += 1 # Remove empty ZIP (no files packed and no missing = nothing to ship) if total_files == 0 and not missing_files: os.unlink(zip_path) # Report " + ".join(p.get("emulator", n) for n, p in selected) missing_count = len(missing_files) ok_count = total_files parts = [f"{ok_count} files packed"] if missing_count: parts.append(f"{missing_count} missing") print(f" {zip_path}: {', '.join(parts)}") for name in missing_files: print(f" MISSING: {name}") for ref in sorted(set(data_dir_notices)): print( f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)" ) return zip_path if total_files > 0 or missing_files else None def generate_system_pack( system_ids: list[str], emulators_dir: str, db: dict, bios_dir: str, output_dir: str, standalone: bool = False, zip_contents: dict | None = None, required_only: bool = False, regions: list[str] | None = None, offline: bool | None = None, ) -> str | None: """Generate a ZIP pack for all emulators supporting given system IDs.""" profiles = load_emulator_profiles(emulators_dir) matching = [] for name, profile in sorted(profiles.items()): if profile.get("type") in ("launcher", "alias", "test"): continue emu_systems = set(profile.get("systems", [])) if emu_systems & set(system_ids): ptype = profile.get("type", "libretro") if standalone and "standalone" not in ptype: continue matching.append(name) if not matching: all_systems: set[str] = set() for p in profiles.values(): all_systems.update(p.get("systems", [])) if standalone: print( f"No standalone emulators found for system(s): {', '.join(system_ids)}", file=sys.stderr, ) else: print( f"No emulators found for system(s): {', '.join(system_ids)}", file=sys.stderr, ) print( f"Available systems: {', '.join(sorted(all_systems)[:20])}...", file=sys.stderr, ) return None # Use system-based ZIP name sys_display = "_".join( "_".join(w.title() for w in sid.split("-")) for sid in system_ids ) result = generate_emulator_pack( matching, emulators_dir, db, bios_dir, output_dir, standalone, zip_contents, required_only=required_only, regions=regions, offline=offline, ) if result: # Rename to system-based name rgn_tag = f"_{region_mod.region_tag(regions)}" if regions else "" new_name = f"{sys_display}{rgn_tag}_BIOS_Pack.zip" new_path = os.path.join(output_dir, new_name) if new_path != result: os.rename(result, new_path) result = new_path return result def list_platforms(platforms_dir: str) -> list[str]: """List available platform names from registry.""" return list_registered_platforms(platforms_dir, include_archived=True) def _system_display_name(system_id: str) -> str: """Convert system ID to display name for ZIP naming.""" s = system_id.lower().replace("_", "-") for prefix in MANUFACTURER_PREFIXES: if s.startswith(prefix): s = s[len(prefix) :] break parts = s.split("-") return "_".join(p.title() for p in parts if p) def _group_systems_by_manufacturer( systems: dict[str, dict], db: dict, bios_dir: str, ) -> dict[str, list[str]]: """Group system IDs by manufacturer for --split --group-by manufacturer.""" from common import derive_manufacturer groups: dict[str, list[str]] = {} for sid, sys_data in systems.items(): mfr = derive_manufacturer(sid, sys_data) groups.setdefault(mfr, []).append(sid) return groups def generate_split_packs( platform_name: str, platforms_dir: str, db: dict, bios_dir: str, output_dir: str, group_by: str = "system", emulators_dir: str = "emulators", zip_contents: dict | None = None, data_registry: dict | None = None, emu_profiles: dict | None = None, target_cores: set[str] | None = None, required_only: bool = False, source: str = "full", regions: list[str] | None = None, target_name: str | None = None, one_per_slot: bool = False, offline: bool | None = None, ) -> list[str]: """Generate split packs (one ZIP per system or manufacturer).""" config = load_platform_config(platform_name, platforms_dir) platform_display = config.get("platform", platform_name) # The split directory carries the same tags as the packs inside it, or two # differently narrowed runs write into one another. split_tags = "".join( tag for tag, _label in _narrowings( source, regions, target_name, one_per_slot, required_only ) ) split_dir = os.path.join( output_dir, f"{platform_display.replace(' ', '_')}{split_tags}_Split" ) os.makedirs(split_dir, exist_ok=True) systems = config.get("systems", {}) if group_by == "manufacturer": groups = _group_systems_by_manufacturer(systems, db, bios_dir) else: groups = {_system_display_name(sid): [sid] for sid in systems} # Pre-compute core extras once (expensive: scans 260+ emulator profiles) # then distribute per group based on emulator system overlap if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) base_dest = config.get("base_destination", "") if source == "platform": all_extras = [] elif emu_profiles: all_extras = _collect_emulator_extras( config, emulators_dir, db, set(), base_dest, emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ) else: all_extras = [] # Extras carry their profile/system identity directly; display labels are # presentation only and must never drive routing. from common import _norm_system_id {_norm_system_id(s): s for s in systems} results = [] for group_name, group_system_ids in sorted(groups.items()): group_sys_set = set(group_system_ids) group_norm = {_norm_system_id(s) for s in group_system_ids} group_match = group_sys_set | group_norm group_extras = [ fe for fe in all_extras if ( set(_extra_system_ids(fe)) | {_norm_system_id(s) for s in _extra_system_ids(fe)} ) & group_match ] zip_path = generate_pack( platform_name, platforms_dir, db, bios_dir, split_dir, emulators_dir=emulators_dir, zip_contents=zip_contents, data_registry=data_registry, emu_profiles=emu_profiles, target_cores=target_cores, required_only=required_only, system_filter=group_system_ids, precomputed_extras=group_extras, source=source, regions=regions, target_name=target_name, one_per_slot=one_per_slot, offline=offline, ) if zip_path: version = config.get("version", config.get("dat_version", "")) ver_tag = f"_{version.replace(' ', '')}" if version else "" narrow_tags = "".join( tag for tag, _label in _narrowings( source, regions, target_name, one_per_slot, required_only ) ) safe_group = group_name.replace(" ", "_") new_name = f"{platform_display.replace(' ', '_')}{ver_tag}{narrow_tags}_{safe_group}_BIOS_Pack.zip" new_path = os.path.join(split_dir, new_name) if new_path != zip_path: os.rename(zip_path, new_path) zip_path = new_path results.append(zip_path) # Warn about extras that couldn't be distributed (emulators without systems: field) all_groups_match = set() for group_system_ids in groups.values(): group_norm = {_norm_system_id(s) for s in group_system_ids} all_groups_match |= set(group_system_ids) | group_norm undistributed = [ fe for fe in all_extras if not ( set(_extra_system_ids(fe)) | {_norm_system_id(s) for s in _extra_system_ids(fe)} ) & all_groups_match ] if undistributed: emus = sorted({fe.get("source_emulator", "?") for fe in undistributed}) print( f" NOTE: {len(undistributed)} core extras from {len(emus)} emulators " f"not in split packs (missing systems: field in profiles: " f"{', '.join(emus[:5])}{'...' if len(emus) > 5 else ''})" ) return results def generate_md5_pack( hashes: list[tuple[str, str]], db: dict, bios_dir: str, output_dir: str, zip_contents: dict | None = None, platform_name: str | None = None, platforms_dir: str | None = None, emulator_name: str | None = None, emulators_dir: str | None = None, standalone: bool = False, ) -> str | None: """Build a pack from an explicit list of hashes with layout context.""" files_db = db.get("files", {}) by_md5 = db.get("indexes", {}).get("by_md5", {}) by_crc32 = db.get("indexes", {}).get("by_crc32", {}) if zip_contents is None: zip_contents = {} plat_file_index: dict[str, dict] = {} base_dest = "" plat_display = "Custom" if platform_name and platforms_dir: config = load_platform_config(platform_name, platforms_dir) base_dest = config.get("base_destination", "") plat_display = config.get("platform", platform_name) for _sys_id, system in config.get("systems", {}).items(): for fe in system.get("files", []): plat_file_index[fe.get("name", "").lower()] = fe emu_pack_structure = None emu_display = "" if emulator_name and emulators_dir: profiles = load_emulator_profiles(emulators_dir, skip_aliases=False) if emulator_name in profiles: profile = profiles[emulator_name] emu_display = profile.get("emulator", emulator_name) emu_pack_structure = profile.get("pack_structure") for fe in profile.get("files", []): plat_file_index[fe.get("name", "").lower()] = fe for alias in fe.get("aliases", []): plat_file_index[alias.lower()] = fe context_name = plat_display if platform_name else (emu_display or "Custom") zip_name = f"{context_name.replace(' ', '_')}_Custom_BIOS_Pack.zip" zip_path = os.path.join(output_dir, zip_name) os.makedirs(output_dir, exist_ok=True) packed: list[tuple[str, str]] = [] not_in_repo: list[tuple[str, str]] = [] not_in_db: list[str] = [] seen: set[str] = set() with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: for hash_type, hash_val in hashes: sha1 = None if hash_type == "sha1" and hash_val in files_db: sha1 = hash_val elif hash_type == "md5": sha1 = by_md5.get(hash_val) elif hash_type == "crc32": sha1 = by_crc32.get(hash_val) if not sha1 or sha1 not in files_db: not_in_db.append(hash_val) continue entry = files_db[sha1] name = entry.get("name", "") aliases = entry.get("aliases") or [] paths = entry.get("paths") or [] dest = name matched_fe = None for lookup_name in [name] + aliases: if lookup_name.lower() in plat_file_index: matched_fe = plat_file_index[lookup_name.lower()] break if matched_fe: if emulator_name and emu_pack_structure is not None: dest = _resolve_destination( matched_fe, emu_pack_structure, standalone ) else: dest = matched_fe.get("destination", matched_fe.get("name", name)) elif paths: dest = paths[0] if base_dest and not dest.startswith(base_dest): full_dest = f"{base_dest}/{dest}" else: full_dest = dest if full_dest in seen: continue seen.add(full_dest) fe_for_resolve = {"name": name, "sha1": sha1, "md5": entry.get("md5", "")} local_path, status = resolve_file( fe_for_resolve, db, bios_dir, zip_contents ) if status == "not_found" or not local_path: not_in_repo.append((name, hash_val)) continue zf.write(local_path, full_dest) packed.append((name, hash_val)) total = len(hashes) print(f"\nPacked {len(packed)}/{total} requested files") for name, h in packed: print(f" PACKED: {name} ({h[:16]}...)") for name, h in not_in_repo: print(f" NOT IN REPO: {name} ({h[:16]}...)") for h in not_in_db: print(f" NOT IN DB: {h}") if not packed: if os.path.exists(zip_path): os.unlink(zip_path) return None return zip_path def generate_target_manifests(targets_dir: str, output_dir: str) -> None: """Convert target YAMLs to installer JSON manifests.""" os.makedirs(output_dir, exist_ok=True) targets_path = Path(targets_dir) if not targets_path.is_dir(): print(f"No targets directory at {targets_dir}", file=sys.stderr) return count = 0 for yml_file in sorted(targets_path.glob("*.yml")): if yml_file.name.startswith("_"): continue with open(yml_file) as f: data = yaml_load(f) or {} targets = data.get("targets", {}) if not isinstance(targets, dict): raise ValueError(f"{yml_file}: targets must be a mapping") # The pack builder accepts the aliases declared in _overrides.yml, so # --target switch works there. Emitting only canonical names left the # installer rejecting the same word and, worse, carrying on with every # file instead: the user asked for a filter and got the full pack. overrides_path = targets_path / "_overrides.yml" alias_map: dict[str, list[str]] = {} if overrides_path.is_file(): with open(overrides_path) as f: all_overrides = yaml_load(f) or {} for tname, ovr in ( all_overrides.get(yml_file.stem, {}).get("targets", {}) or {} ).items(): names = [a for a in (ovr or {}).get("aliases", []) if isinstance(a, str)] if names: alias_map[tname] = names result: dict[str, list[str] | None] = {} for target_name, target_info in targets.items(): if not isinstance(target_name, str) or not target_name: raise ValueError(f"{yml_file}: target names must be non-empty strings") if not isinstance(target_info, dict): raise ValueError(f"{yml_file}: target {target_name!r} must be a mapping") cores = target_info.get("cores") if cores is None or cores == []: result[target_name] = None continue if ( not isinstance(cores, list) or any(not isinstance(core, str) or not core for core in cores) or len(cores) != len(set(cores)) ): raise ValueError( f"{yml_file}: target {target_name!r} cores must be unique " "non-empty strings" ) result[target_name] = cores for target_name, names in alias_map.items(): if target_name not in result: continue for alias in names: if alias in result: raise ValueError( f"{yml_file}: alias {alias!r} collides with a target name" ) result[alias] = result[target_name] out_path = Path(output_dir) / f"{yml_file.stem}.json" with open(out_path, "w") as f: json.dump(result, f, indent=2, sort_keys=True) f.write("\n") count += 1 print(f" {yml_file.stem}: {len(result)} targets") print(f"Generated {count} target manifest(s) in {output_dir}") def _validate_args(args, parser): """Validate argument combinations and mutual exclusion rules.""" has_platform = bool(args.platform) has_all = args.all has_emulator = bool(args.emulator) has_system = bool(args.system) has_from_md5 = bool(args.from_md5 or getattr(args, "from_md5_file", None)) if args.from_md5 and getattr(args, "from_md5_file", None): parser.error("--from-md5 and --from-md5-file are mutually exclusive") if has_from_md5 and has_all: parser.error("--from-md5 requires --platform or --emulator, not --all") if has_from_md5 and has_system: parser.error("--from-md5 and --system are mutually exclusive") if has_from_md5 and args.split: parser.error("--split and --from-md5 are mutually exclusive") # --platform/--all and --system can combine (system filters within platform) # --emulator is exclusive with everything else if has_emulator and (has_platform or has_all or has_system): parser.error( "--emulator is mutually exclusive with --platform, --all, and --system" ) if has_platform and has_all: parser.error("--platform and --all are mutually exclusive") if not (has_platform or has_all or has_emulator or has_system or has_from_md5): parser.error("Specify --platform, --all, --emulator, --system, or --from-md5") if args.standalone and not ( has_emulator or (has_system and not has_platform and not has_all) ): parser.error( "--standalone requires --emulator or --system (without --platform)" ) if args.split and not (has_platform or has_all): parser.error("--split requires --platform or --all") if args.split and has_emulator: parser.error("--split is incompatible with --emulator") if args.group_by != "system" and not args.split: parser.error("--group-by requires --split") if args.target and not (has_platform or has_all): parser.error("--target requires --platform or --all") if args.target and has_emulator: parser.error("--target is incompatible with --emulator") if args.manifest and not (has_platform or has_all): parser.error("--manifest requires --platform or --all") if args.manifest and has_emulator: parser.error("--manifest is incompatible with --emulator") if args.manifest and args.split: parser.error("--manifest is incompatible with --split") if getattr(args, "region", None) and has_from_md5: parser.error("--region and --from-md5 are mutually exclusive") if getattr(args, "one_per_slot", False): if has_from_md5: parser.error("--one-per-slot and --from-md5 are mutually exclusive") if args.manifest: parser.error("--one-per-slot is incompatible with --manifest") if has_emulator or has_system: parser.error("--one-per-slot requires --platform or --all") def _write_manifest_if_changed(path: str, manifest: dict) -> None: """Write manifest JSON only if content (excluding timestamp) changed.""" new_json = json.dumps(manifest, indent=2) if os.path.exists(path): with open(path) as f: try: old = json.load(f) except (json.JSONDecodeError, OSError): old = None if old is not None: # Compare everything except the generated timestamp old_cmp = {k: v for k, v in old.items() if k != "generated"} new_cmp = {k: v for k, v in manifest.items() if k != "generated"} if old_cmp == new_cmp: return # no content change, keep existing timestamp with open(path, "w") as f: f.write(new_json) def _run_manifest_mode( args, groups, db, zip_contents, emu_profiles, target_cores_cache ): """Generate JSON manifests instead of ZIP packs.""" registry_path = os.path.join(args.platforms_dir, "_registry.yml") os.makedirs(args.output_dir, exist_ok=True) registry: dict = {} if os.path.exists(registry_path): with open(registry_path) as _rf: registry = yaml_load(_rf) or {} if args.all_variants: variants = [ ("full", False), ("full", True), ("platform", False), ("platform", True), ("truth", False), ("truth", True), ] else: variants = [(args.source, args.required_only)] for source, required_only in variants: for group_platforms, representative in groups: print(f"\nGenerating manifest for {representative} [source={source}]...") try: tc = target_cores_cache.get(representative) if args.target else None manifest = generate_manifest( representative, args.platforms_dir, db, args.bios_dir, registry_path, emulators_dir=args.emulators_dir, zip_contents=zip_contents, emu_profiles=emu_profiles, target_cores=tc, source=source, regions=getattr(args, "regions", None), target_name=args.target, offline=args.offline, ) narrow_suffix = "".join( tag.lower() for tag, _label in _narrowings( source, getattr(args, "regions", None), args.target, False, required_only, ) ) out_path = os.path.join( args.output_dir, f"{representative}{narrow_suffix}.json", ) _write_manifest_if_changed(out_path, manifest) print( f" {out_path}: {manifest['total_files']} files, " f"{manifest['total_size']} bytes, " f"{manifest['total_omitted']} safely omitted" ) # Create aliases for grouped platforms (e.g., lakka -> retroarch) for alias_plat in group_platforms: if alias_plat != representative: alias_path = os.path.join( args.output_dir, f"{alias_plat}{narrow_suffix}.json", ) alias_manifest = dict(manifest) alias_manifest["platform"] = alias_plat alias_cfg = load_platform_config(alias_plat, args.platforms_dir) alias_manifest["display_name"] = alias_cfg.get( "platform", alias_plat ) alias_registry = registry.get("platforms", {}).get(alias_plat, {}) alias_install = alias_registry.get("install", {}) alias_manifest["detect"] = alias_install.get("detect", []) alias_manifest["standalone_copies"] = alias_install.get( "standalone_copies", [] ) _write_manifest_if_changed(alias_path, alias_manifest) print(f" {alias_path}: alias of {representative}") except (FileNotFoundError, OSError, yaml.YAMLError) as e: print(f" ERROR: {e}") @contextlib.contextmanager def _pack_output_lock(output_dir: str, exclusive: bool = True): """Hold the output directory for the duration of a pack run.""" try: with artifact_lock(output_dir, exclusive=exclusive): yield except ArtifactLockBusy as exc: print(f"ERROR: {exc}") sys.exit(1) def _run_verify_packs(args): """Verify each pack in the output directory against its platform. Delegates to verify_pack_against_platform so the command answers the same question the pipeline asks: baseline files, core extras and structure. A second implementation here answered a narrower one and reported OK while the core extras went unchecked. """ with open(args.db) as f: verify_db = json.load(f) platforms = list_registered_platforms(args.platforms_dir) if args.platform: platforms = [args.platform] elif not args.all: print("ERROR: --verify-packs requires --platform or --all") sys.exit(1) all_ok = True verify_regions = getattr(args, "regions", None) verify_profiles = load_emulator_profiles(args.emulators_dir) verify_data_registry = load_data_dir_registry(args.platforms_dir) for platform_name in platforms: config = load_platform_config(platform_name, args.platforms_dir) display = config.get("platform", platform_name).replace(" ", "_") zip_path = None if os.path.isdir(args.output_dir): for entry in sorted(os.listdir(args.output_dir)): if entry.endswith("_BIOS_Pack.zip") and display in entry: zip_path = os.path.join(args.output_dir, entry) break if not zip_path: print(f" {platform_name}: SKIP (no pack in {args.output_dir})") continue if _narrows_contents(os.path.basename(zip_path)): print(f" {platform_name}: SKIP (narrowed variant)") continue result = verify_pack_against_platform( zip_path, platform_name, args.platforms_dir, db=verify_db, emulators_dir=args.emulators_dir, emu_profiles=verify_profiles, regions=verify_regions, data_registry=verify_data_registry, ) ok, errors = result[0], result[3] bl_checked, bl_present = result[4], result[5] core_checked, core_present = result[6], result[7] counts = ( f"{bl_present}/{bl_checked} baseline, " f"{core_present}/{core_checked} cores" ) if ok: print(f" {platform_name}: OK ({counts})") else: print(f" {platform_name}: FAIL ({counts}, {len(errors)} errors)") for err in errors[:5]: print(f" {err}") all_ok = False if not all_ok: sys.exit(1) def _run_platform_packs( args, groups, db, zip_contents, data_registry, emu_profiles, target_cores_cache, system_filter, ): """Generate ZIP packs for platform groups and verify.""" if args.all_variants: variants = [ ("full", False), ("full", True), ("platform", False), ("platform", True), ("truth", False), ("truth", True), ] else: variants = [(args.source, args.required_only)] for source, required_only in variants: for group_platforms, representative in groups: aliases = [p for p in group_platforms if p != representative] if aliases: all_names = [ load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms ] label = " / ".join(all_names) print(f"\nGenerating pack for {label} [source={source}]...") else: print(f"\nGenerating pack for {representative} [source={source}]...") try: tc = target_cores_cache.get(representative) if args.target else None if args.split: zip_paths = generate_split_packs( representative, args.platforms_dir, db, args.bios_dir, args.output_dir, group_by=args.group_by, emulators_dir=args.emulators_dir, zip_contents=zip_contents, data_registry=data_registry, emu_profiles=emu_profiles, target_cores=tc, required_only=required_only, source=source, regions=getattr(args, "regions", None), target_name=args.target, one_per_slot=args.one_per_slot, offline=args.offline, ) print(f" Split into {len(zip_paths)} packs") else: zip_path = generate_pack( representative, args.platforms_dir, db, args.bios_dir, args.output_dir, include_extras=args.include_extras, emulators_dir=args.emulators_dir, zip_contents=zip_contents, data_registry=data_registry, emu_profiles=emu_profiles, target_cores=tc, required_only=required_only, system_filter=system_filter, source=source, regions=getattr(args, "regions", None), target_name=args.target, one_per_slot=args.one_per_slot, offline=args.offline, ) if not args.split and zip_path and aliases: rep_cfg = load_platform_config(representative, args.platforms_dir) ver = rep_cfg.get("version", rep_cfg.get("dat_version", "")) ver_tag = f"_{ver.replace(' ', '')}" if ver else "" all_names = [ load_platform_config(p, args.platforms_dir).get("platform", p) for p in group_platforms ] narrow_tags = "".join( tag for tag, _label in _narrowings( source, getattr(args, "regions", None), args.target, args.one_per_slot, required_only, ) ) combined = ( "_".join(n.replace(" ", "") for n in all_names) + f"{ver_tag}{narrow_tags}_BIOS_Pack.zip" ) new_path = os.path.join(os.path.dirname(zip_path), combined) if new_path != zip_path: os.rename(zip_path, new_path) print(f" Renamed -> {os.path.basename(new_path)}") except (FileNotFoundError, OSError, yaml.YAMLError) as e: print(f" ERROR: {e}") print("\nVerifying packs and generating manifests...") skip_conf = bool(system_filter or args.split) all_ok = verify_and_finalize_packs( args.output_dir, db, skip_conformance=skip_conf, data_registry=data_registry, regions=getattr(args, "regions", None), ) if args.split: for entry in os.listdir(args.output_dir): sub = os.path.join(args.output_dir, entry) if os.path.isdir(sub) and entry.endswith("_Split"): ok = verify_and_finalize_packs( sub, db, skip_conformance=True, data_registry=data_registry, regions=getattr(args, "regions", None), ) all_ok = all_ok and ok if not all_ok: print("WARNING: some packs have verification errors") sys.exit(1) def main(): global _OFFLINE parser = argparse.ArgumentParser(description="Generate platform BIOS ZIP packs") parser.add_argument("--platform", "-p", help="Platform name (e.g., retroarch)") parser.add_argument( "--all", action="store_true", help="Generate packs for all active platforms" ) parser.add_argument( "--emulator", "-e", help="Emulator profile name(s), comma-separated" ) parser.add_argument("--system", "-s", help="System ID(s), comma-separated") parser.add_argument("--standalone", action="store_true", help="Use standalone mode") parser.add_argument( "--list-emulators", action="store_true", help="List available emulators" ) parser.add_argument( "--list-systems", action="store_true", help="List available systems" ) parser.add_argument( "--include-archived", action="store_true", help="Include archived platforms" ) parser.add_argument("--platforms-dir", default=DEFAULT_PLATFORMS_DIR) parser.add_argument("--db", default=DEFAULT_DB_FILE, help="Path to database.json") parser.add_argument("--bios-dir", default=DEFAULT_BIOS_DIR) parser.add_argument("--output-dir", "-o", default=DEFAULT_OUTPUT_DIR) parser.add_argument( "--include-extras", action="store_true", help="(no-op) Core requirements are always included", ) parser.add_argument("--emulators-dir", default="emulators") parser.add_argument( "--offline", action="store_true", help="Skip data directory freshness check, use cache only", ) parser.add_argument( "--refresh-data", action="store_true", help="Force re-download all data directories", ) parser.add_argument("--list", action="store_true", help="List available platforms") parser.add_argument( "--required-only", action="store_true", help="Only include required files, skip optional", ) parser.add_argument( "--source", choices=["platform", "truth", "full"], default="full", help="File source: platform (YAML only), truth (emulator profiles), full (both)", ) parser.add_argument( "--all-variants", action="store_true", help="Generate all 6 source x required combinations", ) parser.add_argument( "--split", action="store_true", help="Generate one ZIP per system/manufacturer" ) parser.add_argument( "--group-by", choices=["system", "manufacturer"], default="system", help="Grouping for --split (default: system)", ) parser.add_argument("--target", "-t", help="Hardware target (e.g., switch, rpi4)") parser.add_argument( "--region", help="Region priority list, best first (e.g. us,eu,jp)", ) parser.add_argument( "--one-per-slot", action="store_true", help="Keep one BIOS per system and region where the core declares an order", ) parser.add_argument( "--list-targets", action="store_true", help="List available targets for the platform", ) parser.add_argument( "--from-md5", help="Hash(es) to look up or pack (comma-separated)" ) parser.add_argument("--from-md5-file", help="File with hashes (one per line)") parser.add_argument( "--manifest", action="store_true", help="Output JSON manifests instead of ZIP packs", ) parser.add_argument( "--manifest-targets", action="store_true", help="Convert target YAMLs to installer JSON", ) parser.add_argument( "--verify-packs", action="store_true", help="Extract and verify pack integrity (path + hash)", ) args = parser.parse_args() _OFFLINE = bool(args.offline) # Parsed before the quick-exit modes: --verify-packs returns early and # still needs the region priority list to narrow its expectation. args.regions = [] if args.region: try: args.regions = region_mod.parse_requested(args.region) except ValueError as exc: parser.error(str(exc)) if args.manifest_targets: parser.error("--region is incompatible with --manifest-targets") if args.one_per_slot and args.manifest_targets: parser.error("--one-per-slot is incompatible with --manifest-targets") # Quick-exit modes: --verify-packs alone = verify existing packs only # Combined with --all-variants, generation runs first then verify if args.verify_packs and not args.all_variants: with _pack_output_lock(args.output_dir, exclusive=False): _run_verify_packs(args) return if args.manifest_targets: generate_target_manifests( os.path.join(args.platforms_dir, "targets"), args.output_dir ) return if args.list: for p in list_platforms(args.platforms_dir): print(p) return if args.list_emulators: list_emulator_profiles(args.emulators_dir) return if args.list_systems: if args.platform: list_platform_system_ids(args.platform, args.platforms_dir) else: list_system_ids(args.emulators_dir) return if args.list_targets: if not args.platform: parser.error("--list-targets requires --platform") from common import list_available_targets targets = list_available_targets(args.platform, args.platforms_dir) if not targets: print(f"No targets configured for platform '{args.platform}'") return for t in targets: aliases = f" (aliases: {', '.join(t['aliases'])})" if t["aliases"] else "" print( f" {t['name']:30s} {t['architecture']:10s} {t['core_count']:>4d} cores{aliases}" ) return _validate_args(args, parser) # Hash lookup / pack mode has_from_md5 = bool(args.from_md5 or getattr(args, "from_md5_file", None)) if has_from_md5: hashes = ( parse_hash_input(args.from_md5) if args.from_md5 else parse_hash_file(args.from_md5_file) ) if not hashes: print("No valid hashes found in input", file=sys.stderr) sys.exit(1) db = load_database(args.db) if not args.platform and not args.emulator: lookup_hashes( hashes, db, args.bios_dir, args.emulators_dir, args.platforms_dir ) return zip_contents = build_zip_contents_index(db) result = generate_md5_pack( hashes=hashes, db=db, bios_dir=args.bios_dir, output_dir=args.output_dir, zip_contents=zip_contents, platform_name=args.platform, platforms_dir=args.platforms_dir, emulator_name=args.emulator, emulators_dir=args.emulators_dir, standalone=getattr(args, "standalone", False), ) if not result: sys.exit(1) return db = load_database(args.db) zip_contents = build_zip_contents_index(db) # Emulator mode if args.emulator: names = [n.strip() for n in args.emulator.split(",") if n.strip()] if not generate_emulator_pack( names, args.emulators_dir, db, args.bios_dir, args.output_dir, args.standalone, zip_contents, required_only=args.required_only, regions=getattr(args, "regions", None), offline=args.offline, ): sys.exit(1) return # System mode (standalone, without platform context) if args.system and not args.platform and not args.all: system_ids = [s.strip() for s in args.system.split(",") if s.strip()] if not generate_system_pack( system_ids, args.emulators_dir, db, args.bios_dir, args.output_dir, args.standalone, zip_contents, required_only=args.required_only, regions=getattr(args, "regions", None), offline=args.offline, ): sys.exit(1) return system_filter = ( [s.strip() for s in args.system.split(",") if s.strip()] if args.system else None ) # Platform mode if args.all: # Install manifests cover every registered platform. "Archived" means # upstream is no longer scraped on a schedule, not that the platform # lost its users: its packs still ship, so the installer still needs # its file list. platforms = list_registered_platforms( args.platforms_dir, include_archived=args.include_archived or args.manifest, ) elif args.platform: platforms = [args.platform] else: parser.error("Specify --platform or --all") return data_registry = load_data_dir_registry(args.platforms_dir) if data_registry and not args.offline: from refresh_data_dirs import load_registry, refresh_all registry = load_registry(os.path.join(args.platforms_dir, "_data_dirs.yml")) results = refresh_all(registry, force=args.refresh_data) updated = sum(1 for v in results.values() if v) if updated: print(f"Refreshed {updated} data director{'ies' if updated > 1 else 'y'}") emu_profiles = load_emulator_profiles(args.emulators_dir) target_cores_cache: dict[str, set[str] | None] = {} if args.target: try: target_cores_cache, platforms = build_target_cores_cache( platforms, args.target, args.platforms_dir, is_all=args.all, ) except (FileNotFoundError, ValueError) as e: print(f"ERROR: {e}", file=sys.stderr) sys.exit(1) groups = group_identical_platforms( platforms, args.platforms_dir, target_cores_cache if args.target else None ) if args.manifest: _run_manifest_mode( args, groups, db, zip_contents, emu_profiles, target_cores_cache ) else: with _pack_output_lock(args.output_dir): _run_platform_packs( args, groups, db, zip_contents, data_registry, emu_profiles, target_cores_cache, system_filter, ) # Manifest generation (JSON inventory for install.py) _GITIGNORE_ENTRIES: set[str] | None = None def _load_gitignore_entries(repo_root: str) -> set[str]: """Load gitignored paths (large files) from .gitignore at repo root.""" global _GITIGNORE_ENTRIES if _GITIGNORE_ENTRIES is not None: return _GITIGNORE_ENTRIES gitignore = os.path.join(repo_root, ".gitignore") entries: set[str] = set() if os.path.exists(gitignore): with open(gitignore) as f: for line in f: line = line.strip() if line and not line.startswith("#"): entries.add(line) _GITIGNORE_ENTRIES = entries return entries def _is_large_file(local_path: str, repo_root: str) -> bool: """Check if a file is a large file (>50MB or in .gitignore).""" if local_path and os.path.exists(local_path): if os.path.getsize(local_path) > 50_000_000: return True gitignore = _load_gitignore_entries(repo_root) # Check if the path relative to repo root is in .gitignore try: rel = os.path.relpath(local_path, repo_root) except ValueError: rel = "" return rel in gitignore def _get_repo_path(sha1: str, db: dict) -> str: """Get the repo path for a file by SHA1 lookup.""" entry = db.get("files", {}).get(sha1, {}) return entry.get("path", "") def generate_manifest( platform_name: str, platforms_dir: str, db: dict, bios_dir: str, registry_path: str, emulators_dir: str = "emulators", zip_contents: dict | None = None, emu_profiles: dict | None = None, target_cores: set[str] | None = None, source: str = "full", regions: list[str] | None = None, target_name: str | None = None, offline: bool | None = None, ) -> dict: """Generate a JSON manifest for a platform (same resolution as generate_pack). Returns a dict ready for JSON serialization with file inventory, install hints, and download metadata. """ config = load_platform_config(platform_name, platforms_dir) if zip_contents is None: zip_contents = {} if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) platform_display = config.get("platform", platform_name) base_dest = config.get("base_destination", "") case_insensitive = config.get("case_insensitive_fs", False) verification_mode = config.get("verification_mode", "existence") # Load registry for install metadata registry: dict = {} if os.path.exists(registry_path): with open(registry_path) as f: registry = yaml_load(f) or {} plat_registry = registry.get("platforms", {}).get(platform_name, {}) install_section = plat_registry.get("install", {}) detect = install_section.get("detect", []) standalone_copies = install_section.get("standalone_copies", []) repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Filter systems by target from common import resolve_platform_cores plat_cores = resolve_platform_cores(config, emu_profiles) if target_cores else None pack_systems = filter_systems_by_target( config.get("systems", {}), emu_profiles, target_cores, platform_cores=plat_cores, ) seen_destinations: set[str] = set() seen_lower: set[str] = set() seen_parents: set[str] = set() manifest_files: list[dict] = [] omitted_by_destination: dict[str, dict] = {} total_size = 0 def manifest_destination(full_destination: str) -> str: if base_dest and full_destination.startswith(f"{base_dest}/"): return full_destination[len(base_dest) + 1:] return full_destination def record_omission( full_destination: str, file_entry: dict, system_id: str, reason: str, cores: list[str] | None, ) -> None: omitted_by_destination[full_destination] = { "dest": manifest_destination(full_destination), "name": str(file_entry.get("name") or ""), "system": system_id, "required": bool(file_entry.get("required", True)), "reason": reason, "cores": cores, } region_drops: set[str] = set() if regions: region_index = region_mod.build_region_index(emu_profiles) region_groups: dict[str, list[tuple[str, str]]] = {} for sys_id, system in pack_systems.items(): members = region_groups.setdefault(sys_id, []) for file_entry in system.get("files", []): d = _sanitize_path( file_entry.get("destination", file_entry.get("name", "")) ) if d: members.append((d, file_entry.get("name", ""))) if source != "platform": for fe in _collect_emulator_extras( config, emulators_dir, db, set(), base_dest, emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ): d = _sanitize_path(fe.get("destination", fe.get("name", ""))) if not d: continue systems = _extra_system_ids(fe) or ["_extras"] for sid in systems: variant = fe.get("variant_group") group_id = f"{sid}:variant:{variant}" if variant else sid region_groups.setdefault(group_id, []).append( (d, fe.get("name", "")) ) region_drops = region_mod.resolve_region_drops( region_groups, region_index, regions ) # Phase 1: baseline files if source != "truth": for sys_id, system in sorted(pack_systems.items()): for file_entry in system.get("files", []): dest = _sanitize_path(file_entry.get("destination", file_entry["name"])) if not dest: continue if region_drops and dest in region_drops: continue full_dest = f"{base_dest}/{dest}" if base_dest else dest dedup_key = full_dest if dedup_key in seen_destinations: continue if case_insensitive and dedup_key.lower() in seen_lower: continue if _has_path_conflict(full_dest, seen_destinations, seen_parents): continue storage = file_entry.get("storage", "embedded") if storage == "user_provided": record_omission( full_dest, file_entry, sys_id, "user_provided", None, ) continue local_path, status = resolve_file( file_entry, db, bios_dir, zip_contents, offline=offline, ) # An existence platform never reads the bytes, so a declared # hash the local dump contradicts is not a reason to withhold # the file. Hash platforms would reject it, so they omit it. if status in ("not_found", "external") or ( status == "hash_mismatch" and verification_mode != "existence" ): record_omission(full_dest, file_entry, sys_id, status, None) continue # Get SHA1 and size. The installer fetches by hash, so record # the copy this repo holds: an upstream hash carried by no # local file resolves to no download URL at all. sha1 = "" sha256 = "" file_size = 0 if local_path and os.path.exists(local_path): file_size = os.path.getsize(local_path) hashes = compute_hashes(local_path) sha1 = hashes["sha1"] sha256 = hashes["sha256"] repo_path = _get_repo_path(sha1, db) if sha1 else "" is_release_asset = _is_large_file(local_path or "", repo_root) # An entry needs somewhere to be fetched from. Resolution can # land on a file the database does not index -- a data # directory, or a copy added since the last scan -- and then # neither a repo path nor a release asset exists, so the # installer would list a file it can never download. Recording # it as omitted keeps that visible instead of shipping a dead # entry. if not repo_path and not is_release_asset: record_omission(full_dest, file_entry, sys_id, "not_found", None) continue entry: dict = { "dest": dest, "sha1": sha1, "sha256": sha256, "size": file_size, "repo_path": repo_path, "cores": None, } if is_release_asset: entry["storage"] = "release" entry["release_asset"] = ( os.path.basename(local_path) if local_path else file_entry["name"] ) manifest_files.append(entry) omitted_by_destination.pop(full_dest, None) total_size += file_size seen_destinations.add(dedup_key) _register_path(dedup_key, seen_destinations, seen_parents) if case_insensitive: seen_lower.add(dedup_key.lower()) # Phase 2: core complement (emulator extras) if source != "platform": core_files = _collect_emulator_extras( config, emulators_dir, db, seen_destinations, base_dest, emu_profiles, target_cores=target_cores, include_all=(source == "truth"), ) else: core_files = [] extras_pfx = _detect_extras_prefix(config, base_dest) for fe in core_files: dest = _sanitize_path(fe.get("destination", fe["name"])) if not dest: continue if region_drops and dest in region_drops: continue if extras_pfx: if not dest.startswith(f"{extras_pfx}/"): full_dest = f"{extras_pfx}/{dest}" else: full_dest = dest else: full_dest = dest if full_dest in seen_destinations: continue if case_insensitive and full_dest.lower() in seen_lower: continue if _has_path_conflict(full_dest, seen_destinations, seen_parents): continue dest_hint = fe.get("destination", "") local_path, status = resolve_file( fe, db, bios_dir, zip_contents, dest_hint=dest_hint, offline=offline, ) if status in ("not_found", "external", "user_provided") or not local_path: source_emu = fe.get("source_profile") or fe.get("source_emulator", "") systems = _extra_system_ids(fe) record_omission( full_dest, fe, systems[0] if systems else "", status, [source_emu] if source_emu else [], ) continue sha1 = "" sha256 = "" file_size = 0 if local_path and os.path.exists(local_path): file_size = os.path.getsize(local_path) hashes = compute_hashes(local_path) sha1 = hashes["sha1"] sha256 = hashes["sha256"] repo_path = _get_repo_path(sha1, db) if sha1 else "" source_emu = fe.get("source_profile") or fe.get("source_emulator", "") # Manifest dests are relative to base_destination; keep the inferred # extras prefix when it is an internal layout dir (RetroDECK bios/). manifest_dest = full_dest if base_dest and manifest_dest.startswith(f"{base_dest}/"): manifest_dest = manifest_dest[len(base_dest) + 1:] entry = { "dest": manifest_dest, "sha1": sha1, "sha256": sha256, "size": file_size, "repo_path": repo_path, "cores": [source_emu] if source_emu else [], } if _is_large_file(local_path or "", repo_root): entry["storage"] = "release" entry["release_asset"] = ( os.path.basename(local_path) if local_path else fe["name"] ) manifest_files.append(entry) omitted_by_destination.pop(full_dest, None) total_size += file_size seen_destinations.add(full_dest) _register_path(full_dest, seen_destinations, seen_parents) if case_insensitive: seen_lower.add(full_dest.lower()) # No phase 3 (data directories) -skipped for manifest now = _build_timestamp(db) result: dict = { "manifest_version": 2, "source": source, "regions": list(regions or []), # A target-filtered manifest is only distinguishable from the full one # by its filename, so it records the filter the way regions does. # Absent on an unfiltered manifest rather than empty, to leave the # twelve default files byte-identical. **({"target": target_name} if target_name else {}), "platform": platform_name, "display_name": platform_display, "version": "1.0", "generated": now, "base_destination": base_dest, "detect": detect, "standalone_copies": standalone_copies, "total_files": len(manifest_files), "total_size": total_size, "total_omitted": len(omitted_by_destination), "omitted_files": sorted( omitted_by_destination.values(), key=lambda entry: entry["dest"] ), "files": manifest_files, } return result # Post-generation pack verification + manifest + SHA256SUMS def verify_pack( zip_path: str, db: dict, data_registry: dict | None = None ) -> tuple[bool, dict]: """Verify a generated pack ZIP by re-hashing every file inside. Checks against database.json, data directory caches, and verifies rebuilt ZIP content by comparing inner CRC32s against source. Returns (all_ok, manifest_dict). """ files_db = db.get("files", {}) # SHA1 -> file_info by_md5 = db.get("indexes", {}).get("by_md5", {}) # MD5 -> SHA1 by_name = db.get("indexes", {}).get("by_name", {}) # name -> [SHA1] # Data directory file index _data_index: dict[str, list[str]] = {} _data_path_index: dict[str, str] = {} if data_registry: for _dk, _de in data_registry.items(): cache = _de.get("local_cache", "") if not cache or not os.path.isdir(cache): continue for _r, _d, _fns in os.walk(cache): for _fn in _fns: _fp = os.path.join(_r, _fn) _rel = os.path.relpath(_fp, cache) _data_path_index[_rel] = _fp _data_index.setdefault(_fn, []).append(_fp) manifest = { "schema_version": 1, "version": 1, "generator": "retrobios generate_pack.py", "generated": _build_timestamp(db), "files": [], } errors = [] with zipfile.ZipFile(zip_path, "r") as zf: for info in zf.infolist(): if info.is_dir(): continue name = info.filename if name.startswith("INSTRUCTIONS_") or name in ( "manifest.json", "README.txt", ): continue with zf.open(info) as f: sha1_h = hashlib.sha1() md5_h = hashlib.md5() size = 0 for chunk in iter(lambda: f.read(65536), b""): sha1_h.update(chunk) md5_h.update(chunk) size += len(chunk) sha1 = sha1_h.hexdigest() md5 = md5_h.hexdigest() # Look up in database: files_db keyed by SHA1 db_entry = files_db.get(sha1) status = "verified" file_name = "" if db_entry: file_name = db_entry.get("name", "") else: # Try MD5 -> SHA1 lookup ref_sha1 = by_md5.get(md5) if ref_sha1: db_entry = files_db.get(ref_sha1) if db_entry: file_name = db_entry.get("name", "") status = "verified_md5" else: status = "untracked" else: status = "untracked" # Rebuilt ZIP: verify inner ROM CRC32s match source if status == "untracked" and name.endswith(".zip"): _bn = os.path.basename(name) for _src_sha1 in by_name.get(_bn, []): if _src_sha1 not in files_db: continue _src_path = files_db[_src_sha1]["path"] if not os.path.exists(_src_path): continue try: import io as _io with zipfile.ZipFile(_src_path) as _sz: _sc = { i.filename: i.CRC for i in _sz.infolist() if not i.is_dir() } with zipfile.ZipFile(_io.BytesIO(zf.read(name))) as _pz: _pc = { i.filename: i.CRC for i in _pz.infolist() if not i.is_dir() } if _sc == _pc: status = "verified_rebuild" file_name = _bn break except (zipfile.BadZipFile, OSError): continue # Data directory: check against cached files if status == "untracked" and _data_index: _bn = os.path.basename(name) _pr = name for _known_prefix in ("system/", "bios/", "BIOS/", "Firmware/"): if name.startswith(_known_prefix): _pr = name[len(_known_prefix):] break _cands = [] if _pr in _data_path_index: _cands.append(_data_path_index[_pr]) for _dp in _data_index.get(_bn, []): if _dp not in _cands: _cands.append(_dp) for _dp in _cands: if not os.path.exists(_dp): continue if os.path.getsize(_dp) == size: status = "verified_data" file_name = _bn break if name.endswith(".zip") and _dp.endswith(".zip"): try: import io as _io2 with zipfile.ZipFile(_io2.BytesIO(zf.read(name))) as _pz2: _pc2 = { i.filename: i.CRC for i in _pz2.infolist() if not i.is_dir() } with zipfile.ZipFile(_dp) as _dz: _dc = { i.filename: i.CRC for i in _dz.infolist() if not i.is_dir() } if _pc2 == _dc: status = "verified_data" file_name = _bn break except (zipfile.BadZipFile, OSError): continue manifest["files"].append( { "path": name, "sha1": sha1, "md5": md5, "size": size, "status": status, "name": file_name, } ) # Corruption check: SHA1 in DB but doesn't match what we computed # This should never happen (we looked up by SHA1), but catches # edge cases where by_md5 resolved to a different SHA1 if db_entry and status == "verified_md5": expected_sha1 = db_entry.get("sha1", "") if expected_sha1 and expected_sha1.lower() != sha1.lower(): errors.append( f"{name}: SHA1 mismatch (expected {expected_sha1}, got {sha1})" ) verified = sum(1 for f in manifest["files"] if f["status"].startswith("verified")) untracked = sum(1 for f in manifest["files"] if f["status"] == "untracked") total = len(manifest["files"]) manifest["summary"] = { "total_files": total, "verified": verified, "untracked": untracked, "errors": len(errors), } manifest["errors"] = errors all_ok = len(errors) == 0 return all_ok, manifest def inject_manifest(zip_path: str, manifest: dict) -> None: """Inject manifest.json into an existing ZIP pack.""" manifest_json = json.dumps(manifest, indent=2, ensure_ascii=False) # Check if manifest already exists with zipfile.ZipFile(zip_path, "r") as zf: has_manifest = "manifest.json" in zf.namelist() if not has_manifest: # Fast path: append directly. Append mode defaults to ZIP_STORED, # which would leave the manifest uncompressed and give the same pack # two different byte layouts depending on the path taken here. with zipfile.ZipFile(zip_path, "a", zipfile.ZIP_DEFLATED) as zf: _write_generated_member(zf, "manifest.json", manifest_json) else: # Rebuild to replace existing manifest import tempfile as _tempfile tmp_fd, tmp_path = _tempfile.mkstemp( suffix=".zip", dir=os.path.dirname(zip_path) ) os.close(tmp_fd) try: with ( zipfile.ZipFile(zip_path, "r") as src, zipfile.ZipFile(tmp_path, "w", zipfile.ZIP_DEFLATED) as dst, ): for item in src.infolist(): if item.filename == "manifest.json": continue dst.writestr(item, src.read(item.filename)) _write_generated_member(dst, "manifest.json", manifest_json) os.replace(tmp_path, zip_path) except (OSError, zipfile.BadZipFile): os.unlink(tmp_path) raise def generate_sha256sums(output_dir: str) -> str | None: """Generate SHA256SUMS.txt for all ZIP files in output_dir.""" sums_path = os.path.join(output_dir, "SHA256SUMS.txt") entries = [] for name in sorted(os.listdir(output_dir)): if not name.endswith(".zip"): continue path = os.path.join(output_dir, name) sha256 = hashlib.sha256() with open(path, "rb") as f: for chunk in iter(lambda: f.read(65536), b""): sha256.update(chunk) entries.append(f"{sha256.hexdigest()} {name}") if not entries: return None with open(sums_path, "w") as f: f.write("\n".join(entries) + "\n") print(f"\n{sums_path}: {len(entries)} pack checksums") return sums_path def _hash_matches(declared: str, actual: str) -> bool: """Compare a declared hash value against an actual hex digest. Handles comma-separated multi-hash lists and uppercase (Recalbox) and truncated MD5s (Batocera 29-char): a declared value shorter than the digest matches by prefix. """ actual = actual.lower() for cand in declared.split(","): cand = cand.strip().lower() if not cand: continue if len(cand) < len(actual): if actual.startswith(cand): return True elif actual == cand: return True return False def _check_member_hash( zf: zipfile.ZipFile, member: str, file_entry: dict, mode: str ) -> str | None: """Verify a pack member against its platform-declared hash. Reproduces the platform's native check inside the pack: md5/sha1 of the member bytes, or checkInsideZip when zipped_file is set (Batocera hashes a ROM inside the ZIP, matched case-insensitively). Returns an error string, or None when the member passes. """ declared = str(file_entry.get(mode) or "").strip() if not declared: return None zipped_file = file_entry.get("zipped_file") if zipped_file: import io try: with zipfile.ZipFile(io.BytesIO(zf.read(member))) as inner: want = zipped_file.casefold() target = next( (n for n in inner.namelist() if n.casefold() == want), None ) if target is None: return f"{member}: {zipped_file} not found inside ZIP" actual = hashlib.md5(inner.read(target)).hexdigest() except zipfile.BadZipFile: return f"{member}: not a valid ZIP" if not _hash_matches(declared, actual): return ( f"{member}: {zipped_file} inside-zip md5 {actual} " f"!= declared {declared}" ) return None h = hashlib.md5() if mode == "md5" else hashlib.sha1() with zf.open(member) as f: for chunk in iter(lambda: f.read(65536), b""): h.update(chunk) actual = h.hexdigest() if _hash_matches(declared, actual): return None # Recalbox Md5Composite: MD5 over sorted inner contents of a ZIP, # independent of compression and metadata (Zip::Md5Composite()). if mode == "md5" and member.endswith(".zip"): import io try: with zipfile.ZipFile(io.BytesIO(zf.read(member))) as inner: names = sorted(n for n in inner.namelist() if not n.endswith("/")) ch = hashlib.md5() for n in names: ch.update(inner.read(n)) if _hash_matches(declared, ch.hexdigest()): return None except (zipfile.BadZipFile, OSError): pass return f"{member}: {mode} {actual} != declared {declared}" def _repo_satisfies_declaration( entries: list[dict], db: dict, mode: str ) -> bool: """Check whether any repo file matches one of the declared hashes. Used to separate pack divergence (a matching file exists but was not packed) from data coverage gaps (no repo file matches the upstream declaration): only the former is a pack generation error. """ from common import md5_composite files_db = db.get("files", {}) by_md5 = db.get("indexes", {}).get("by_md5", {}) by_name = db.get("indexes", {}).get("by_name", {}) for fe in entries: declared = str(fe.get(mode) or "").strip() if not declared: continue hashes = [h.strip().lower() for h in declared.split(",") if h.strip()] for h in hashes: if mode == "sha1": entry = files_db.get(h) else: entry = files_db.get(by_md5.get(h, "")) if entry and os.path.exists(entry.get("path", "")): return True if mode == "md5": hash_set = set(hashes) for cand_sha in by_name.get(fe.get("name", ""), []): entry = files_db.get(cand_sha) if not entry: continue path = entry.get("path", "") if not path.endswith(".zip") or not os.path.exists(path): continue try: if md5_composite(path).lower() in hash_set: return True except (zipfile.BadZipFile, OSError): continue return False def _intentional_hash_exclusion( entries: list[dict], db: dict, bios_dir: str = DEFAULT_BIOS_DIR, zip_contents: dict | None = None, data_dir_registry: dict | None = None, verification_mode: str = "md5", ) -> bool: """Return whether the builder must omit every declaration as unsafe. A hash platform never substitutes a same-named payload for an explicitly hash-identified one, so a missing pack member is accounted for when every declaration for that destination resolves to a local hash mismatch. An existence platform reads no bytes and the builder ships the file anyway, so nothing is ever excluded on its behalf. Genuine absence, external-download failure and a packable alternative stay conformance errors in both modes. """ if not entries or not hash_mismatch_excludes_file(verification_mode): return False archive_index = zip_contents if zip_contents is not None else {} for entry in entries: local_path, status = resolve_file( entry, db, bios_dir, archive_index, data_dir_registry=data_dir_registry, offline=True, ) if status != "hash_mismatch": return False # A container can mismatch the outer declaration while still carrying # the exact inner ROM requested by Batocera-style zipped_file entries. zipped_file = entry.get("zipped_file") if zipped_file and local_path: declared = str(entry.get("md5") or "") candidates = [value.strip() for value in declared.split(",") if value.strip()] if not candidates: candidates = [""] if any( check_inside_zip(local_path, zipped_file, candidate) == "ok" for candidate in candidates ): return False return True def verify_pack_against_platform( zip_path: str, platform_name: str, platforms_dir: str, db: dict | None = None, emulators_dir: str = "emulators", emu_profiles: dict | None = None, regions: list[str] | None = None, data_registry: dict | None = None, ) -> tuple[bool, int, int, list[str], int, int, int, int, int]: """Verify a pack ZIP against its platform config and core requirements. A region priority list narrows the expectation to what the builder would have packed, using the same selection function. Checks: 1. Every baseline file declared by the platform exists in the ZIP at the correct destination path 2. Every in-repo core extra file (from emulator profiles) is present 3. No duplicate entries 4. No path anomalies (double slash, absolute, traversal) 5. No unexpected zero-byte BIOS files Returns ``(all_ok, checked, present, errors, baseline_checked, baseline_present, core_checked, core_present, baseline_excluded)``. """ from collections import Counter config = load_platform_config(platform_name, platforms_dir) base_dest = config.get("base_destination", "") errors: list[str] = [] checked = 0 present = 0 if emu_profiles is None: emu_profiles = load_emulator_profiles(emulators_dir) region_drops: set[str] = set() if regions: region_index = region_mod.build_region_index(emu_profiles) region_groups: dict[str, list[tuple[str, str]]] = {} for sys_id, system in config.get("systems", {}).items(): members = region_groups.setdefault(sys_id, []) for fe in system.get("files", []): d = _sanitize_path(fe.get("destination", fe.get("name", ""))) if d: members.append((d, fe.get("name", ""))) if db is not None: for extra in _collect_emulator_extras( config, emulators_dir, db, set(), base_dest, emu_profiles, ): d = _sanitize_path( extra.get("destination", extra.get("name", "")) ) if not d: continue systems_for_extra = _extra_system_ids(extra) or ["_extras"] for sys_id in systems_for_extra: variant = extra.get("variant_group") group_id = ( f"{sys_id}:variant:{variant}" if variant else sys_id ) region_groups.setdefault(group_id, []).append( (d, extra.get("name", "")) ) region_drops = region_mod.resolve_region_drops( region_groups, region_index, regions ) with zipfile.ZipFile(zip_path, "r") as zf: zip_set = set(zf.namelist()) zip_lower = {n.lower(): n for n in zip_set} # Auto-detect flat vs nested ZIP is_flat = bool(base_dest) and not any( n.startswith(base_dest + "/") for n in zip_set if n not in ("README.txt", "manifest.json") and not n.endswith("/") ) # Structural checks dupes = sum(1 for c in Counter(zf.namelist()).values() if c > 1) if dupes: errors.append(f"{dupes} duplicate entries") for n in zip_set: if "//" in n: errors.append(f"double slash: {n}") if n.startswith("/"): errors.append(f"absolute path: {n}") if ".." in n: errors.append(f"path traversal: {n}") # Zero-byte check (exclude Dolphin GraphicMods markers) for info in zf.infolist(): if info.file_size == 0 and not info.is_dir(): if "GraphicMods" not in info.filename and info.filename not in ( "manifest.json", "README.txt", ): errors.append(f"zero-byte: {info.filename}") # 1. Baseline file presence + native hash check verification_mode = config.get("verification_mode", "existence") baseline_checked = 0 baseline_present = 0 baseline_excluded = 0 decl_by_member: dict[str, list[dict]] = {} # Mirror the builder's path-conflict logic: a declaration whose path # collides file-vs-directory with a packed member was skipped by the # builder (upstream declares e.g. both SGB1.sfc and SGB1.sfc/program.rom) zip_parents: set[str] = set() for n in zip_set: parts = n.split("/") for i in range(1, len(parts)): zip_parents.add("/".join(parts[:i])) baseline_groups: dict[str, list[dict]] = {} for _sys_id, system in config.get("systems", {}).items(): for fe in system.get("files", []): dest = _sanitize_path(fe.get("destination", fe.get("name", ""))) if not dest: continue if region_drops and dest in region_drops: continue expected = f"{base_dest}/{dest}" if base_dest and not is_flat else dest baseline_groups.setdefault(expected, []).append(fe) baseline_checked = len(baseline_groups) exclusion_index = build_zip_contents_index(db) if db is not None else {} for expected, declarations in baseline_groups.items(): if expected in zip_set: member = expected elif expected.lower() in zip_lower: member = zip_lower[expected.lower()] elif _has_path_conflict(expected, zip_set, zip_parents): # Skipped by the builder for the same reason: not an error baseline_present += 1 continue elif db is not None and _intentional_hash_exclusion( declarations, db, zip_contents=exclusion_index, data_dir_registry=data_registry, verification_mode=verification_mode, ): baseline_excluded += 1 continue else: errors.append(f"baseline missing: {expected}") continue baseline_present += 1 decl_by_member.setdefault(member, []).extend(declarations) # Reproduce the platform's native hash check on pack bytes. # A destination declared by several entries passes when the packed # member satisfies any of them. A failure only counts as a pack # error when the repo holds a file matching a declaration: without # one, the pack ships its best effort and the gap is a data issue # reported by verify.py, not a generation bug. if verification_mode in ("md5", "sha1"): for member, decl_entries in decl_by_member.items(): checkable = [ fe for fe in decl_entries if str(fe.get(verification_mode) or "").strip() ] if not checkable: continue member_errors = [] satisfied = False for fe in checkable: err = _check_member_hash(zf, member, fe, verification_mode) if err is None: satisfied = True break member_errors.append(err) if satisfied: continue if db is None or _repo_satisfies_declaration( checkable, db, verification_mode ): errors.append(member_errors[0]) # 2. Core extras presence (files from emulator profiles, in repo) # Mirror the pack builder's skip logic: only count files that # can actually be resolved and don't have path conflicts. core_checked = 0 core_present = 0 if db is not None: core_files = _collect_emulator_extras( config, emulators_dir, db, set(), base_dest, emu_profiles, ) seen_conformance: set[str] = set(zip_set) seen_parents: set[str] = set() for n in zip_set: parts = n.split("/") for i in range(1, len(parts)): seen_parents.add("/".join(parts[:i])) extras_pfx = _detect_extras_prefix(config, base_dest) for fe in core_files: raw_dest = fe.get("destination", fe.get("name", "")) dest = _sanitize_path(raw_dest) if not dest: continue if region_drops and _sanitize_path(dest) in region_drops: continue if extras_pfx and not (is_flat and extras_pfx == base_dest): if not dest.startswith(f"{extras_pfx}/"): full = f"{extras_pfx}/{dest}" else: full = dest else: full = dest # Skip path conflicts (same logic as pack builder) if _has_path_conflict(full, seen_conformance, seen_parents): continue # Skip unresolvable files (game_data dirs, etc.) local_path, status = resolve_file( fe, db, "bios", {}, dest_hint=raw_dest, data_dir_registry=data_registry, offline=True, ) if status in ("not_found", "external", "user_provided"): continue core_checked += 1 if full in zip_set or full.lower() in zip_lower: core_present += 1 seen_conformance.add(full) _register_path(full, seen_conformance, seen_parents) else: errors.append(f"core missing: {full}") checked = baseline_checked + core_checked present = baseline_present + core_present return ( len(errors) == 0, checked, present, errors, baseline_checked, baseline_present, core_checked, core_present, baseline_excluded, ) def _narrows_contents(pack_name: str) -> bool: """True when a pack holds fewer files than the platform declares. A source-restricted or required-only build is narrower by design, so the full platform expectation does not apply to it and conformance is skipped. Region is not listed: the region filter is passed to the check itself. """ return any( tag in pack_name for tag in ("_Platform_", "_Truth_", "_Required", "_OnePerSlot") ) def verify_and_finalize_packs( output_dir: str, db: dict, platforms_dir: str = "platforms", skip_conformance: bool = False, data_registry: dict | None = None, regions: list[str] | None = None, ) -> bool: """Verify all packs, inject manifests, generate SHA256SUMS. Two-stage verification: 1. Hash check against database.json (integrity) 2. Extract + verify against platform config (conformance) Returns True if all packs pass verification. """ all_ok = True # Map ZIP names to platform names pack_to_platform: dict[str, list[str]] = {} for name in sorted(os.listdir(output_dir)): if not name.endswith(".zip"): continue for pname in list_registered_platforms(platforms_dir): cfg = load_platform_config(pname, platforms_dir) display = cfg.get("platform", pname).replace(" ", "_") if display in name or display.replace("_", "") in name.replace("_", ""): pack_to_platform.setdefault(name, []).append(pname) for name in sorted(os.listdir(output_dir)): if not name.endswith(".zip"): continue zip_path = os.path.join(output_dir, name) # Stage 1: database integrity ok, manifest = verify_pack(zip_path, db, data_registry=data_registry) summary = manifest["summary"] status = "OK" if ok else "ERRORS" print( f" verify {name}: {summary['verified']}/{summary['total_files']} verified, " f"{summary['untracked']} untracked, {summary['errors']} errors [{status}]" ) if not ok: for err in manifest["errors"]: print(f" ERROR: {err}") all_ok = False inject_manifest(zip_path, manifest) # Stage 2: platform conformance (extract + verify) # Skipped for filtered/split/custom packs (intentionally partial) if skip_conformance: continue if _narrows_contents(name): continue platforms = pack_to_platform.get(name, []) for pname in platforms: ( p_ok, total, matched, p_errors, bl_checked, bl_present, core_checked, core_present, bl_excluded, ) = verify_pack_against_platform( zip_path, pname, platforms_dir, db=db, regions=regions, data_registry=data_registry, ) status = "OK" if p_ok else "FAILED" exclusion_note = ( f", {bl_excluded} unsafe excluded" if bl_excluded else "" ) print( f" platform {pname}: {bl_present}/{bl_checked} baseline present" f"{exclusion_note}, " f"{core_present}/{core_checked} cores present, {status}" ) if not p_ok: for err in p_errors: print(f" {err}") all_ok = False generate_sha256sums(output_dir) return all_ok if __name__ == "__main__": main()