kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-10-11 05:53:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: decide hash mismatch by native mode

Browse Source 
A declared hash that the local dump contradicts is not one situation. An
existence platform never reads the bytes, so withholding the file lets an
upstream list error remove something the frontend would have loaded; a
hash platform would reject it, so shipping it is pointless.

The mode now decides, at every point that had an opinion: pack building,
core complement, emulator packs, manifests, conformance and
_intentional_hash_exclusion. verify.find_undeclared_files follows, since
verify and generate_pack must agree file for file.

Also here: resolution reports which evidence matched rather than a flat
"exact", a path or filename can no longer override a declared hash, and
safe_extract_zip treats a Windows backslash as the separator it is
instead of refusing the archive.
This commit is contained in:
Abdessamad Derraz committed 2026-08-10 13:36:03 +02:00
1 parent a5f549a1c6
commit e8ee8b0954
12 files changed
+1805 -269
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/common.py
+298 -53
View File
@@ -10,6 +10,8 @@ import contextlib
import hashlib
import json
import os
import re
import stat
import tempfile
import urllib.error
import urllib.parse
@@ -421,6 +423,24 @@ def list_available_targets(
return result
HASH_EXACT_RESOLUTION_STATUSES = frozenset(
{
"sha1_exact",
"sha256_exact",
"crc32_exact",
"md5_exact",
"md5_composite_exact",
"zip_exact",
"data_dir_hash_exact",
}
)
def resolution_is_hash_exact(status: str) -> bool:
"""Whether *status* proves content identity with a declared hash."""
return status in HASH_EXACT_RESOLUTION_STATUSES
def resolve_local_file(
file_entry: dict,
db: dict,
@@ -439,8 +459,12 @@ def resolve_local_file(
disambiguate when multiple files share the same name. Matched against
the by_path_suffix index built from the repo's directory structure.
Returns (local_path, status) where status is one of:
exact, zip_exact, hash_mismatch, not_found.
Returns ``(local_path, status)``. Statuses describe the evidence used:
``sha1_exact``, ``sha256_exact``, ``crc32_exact``, ``md5_exact``,
``md5_composite_exact``, ``zip_exact``, ``path_exact``, ``name_exact``,
``hash_mismatch`` or ``not_found`` (plus documented fallback statuses).
A path or filename is never allowed to override a declared strong hash.
"""
sha1 = file_entry.get("sha1")
name = file_entry.get("name", "")
@@ -464,54 +488,100 @@ def resolve_local_file(
names_to_try.append(hint_base)
md5_list = parse_md5_list(file_entry.get("md5"))
sha1_candidates = [
str(value).strip().lower()
for value in (sha1 if isinstance(sha1, list) else [sha1] if sha1 else [])
if str(value).strip()
]
sha256_raw = file_entry.get("sha256")
sha256_values = sha256_raw if isinstance(sha256_raw, list) else [sha256_raw]
sha256_candidates = [
candidate.strip().lower()
for value in sha256_values
if value
for candidate in str(value).split(",")
if len(candidate.strip()) == 64
]
crc_raw = str(file_entry.get("crc32", "") or "").strip().lower()
declared_size = file_entry.get("size")
has_strong_hash = bool(
sha1_candidates or sha256_candidates or md5_list or crc_raw
)
files_db = db.get("files", {})
by_md5 = db.get("indexes", {}).get("by_md5", {})
by_name = db.get("indexes", {}).get("by_name", {})
by_path_suffix = db.get("indexes", {}).get("by_path_suffix", {})
# 0. Path suffix exact match (for regional variants with same filename)
if dest_hint and by_path_suffix:
for match_sha1 in by_path_suffix.get(dest_hint, []):
if match_sha1 in files_db:
path = files_db[match_sha1]["path"]
if os.path.exists(path):
return path, "exact"
def _record_match_status(match_sha1: str) -> str | None:
"""Return hash evidence when a DB record satisfies every declaration."""
entry = files_db.get(match_sha1)
if not entry:
return None
statuses: list[str] = []
if sha1_candidates:
if match_sha1.lower() not in sha1_candidates:
return None
statuses.append("sha1_exact")
if sha256_candidates:
if str(entry.get("sha256", "")).lower() not in sha256_candidates:
return None
statuses.append("sha256_exact")
# With zipped_file the MD5 identifies the member, not the container.
if md5_list and not zipped_file:
actual_md5 = str(entry.get("md5", "")).lower()
if not any(actual_md5.startswith(expected) for expected in md5_list):
return None
statuses.append("md5_exact")
if crc_raw:
if str(entry.get("crc32", "")).lower() != crc_raw:
return None
if declared_size is not None:
allowed_sizes = (
declared_size if isinstance(declared_size, list) else [declared_size]
)
if entry.get("size") not in allowed_sizes:
return None
statuses.append("crc32_exact")
return statuses[0] if statuses else None
# 1. SHA1 exact match (accept list-valued sha1 from profiles)
sha1_candidates = sha1 if isinstance(sha1, list) else [sha1] if sha1 else []
for cand in sha1_candidates:
if isinstance(cand, str) and cand in files_db:
if cand in files_db:
path = files_db[cand]["path"]
if os.path.exists(path):
return path, "exact"
status = _record_match_status(cand)
if status and os.path.exists(path):
return path, status
# 1b. SHA256 exact match (profiles hashed from sources that publish
# sha256, e.g. MesenCE). A full sha256 is a strong identifier.
sha256_raw = str(file_entry.get("sha256", "") or "")
if sha256_raw:
if sha256_candidates:
by_sha256 = db.get("indexes", {}).get("by_sha256", {})
for cand in sha256_raw.split(","):
cand = cand.strip().lower()
if len(cand) == 64:
match = by_sha256.get(cand)
if match and match in files_db:
path = files_db[match]["path"]
if os.path.exists(path):
return path, "exact"
for cand in sha256_candidates:
match = by_sha256.get(cand)
if match and match in files_db:
path = files_db[match]["path"]
status = _record_match_status(match)
if status and os.path.exists(path):
return path, status
# 1c. CRC32 + size exact match, only when no stronger hash is declared
# (crc-only profiles: FBNeo, Clock Signal). CRC32 alone is weak, so the
# declared size must confirm the match.
crc_raw = str(file_entry.get("crc32", "") or "").strip().lower()
declared_size = file_entry.get("size")
if crc_raw and declared_size and not zipped_file and not md5_list:
# 1c. CRC32 lookup, only when no stronger hash is declared. A declared
# size confirms it when available; a handful of emulators validate CRC32
# alone, so those entries retain the same (weaker) evidence as the core.
if (
crc_raw
and not zipped_file
and not md5_list
and not sha1_candidates
and not sha256_candidates
):
by_crc32 = db.get("indexes", {}).get("by_crc32", {})
match = by_crc32.get(crc_raw)
if match and match in files_db:
entry = files_db[match]
path = entry["path"]
if entry.get("size") == declared_size and os.path.exists(path):
return path, "exact"
status = _record_match_status(match)
if status and os.path.exists(path):
return path, status
# 2. MD5 direct lookup (skip for zipped_file: md5 is inner ROM, not container)
# Guard: only accept if the found file's name matches the requested name
@@ -534,15 +604,31 @@ def resolve_local_file(
# Full MD5 (32 chars) is a strong identifier: trust it
# without name guard. Truncated MD5 still needs name check
# to avoid cross-contamination.
if os.path.exists(path):
status = _record_match_status(sha1_match)
if status and os.path.exists(path):
if len(md5_candidate) >= 32 or _md5_name_ok(path):
return path, "md5_exact"
return path, status
if len(md5_candidate) < 32:
for db_md5, db_sha1 in by_md5.items():
if db_md5.startswith(md5_candidate) and db_sha1 in files_db:
path = files_db[db_sha1]["path"]
if os.path.exists(path) and _md5_name_ok(path):
return path, "md5_exact"
status = _record_match_status(db_sha1)
if status and os.path.exists(path) and _md5_name_ok(path):
return path, status
# 2b. Path suffix lookup is useful for same-named regional files, but it
# is identity evidence only when no content hash was declared. A stale
# or incorrect destination can therefore never mask a hash mismatch.
if dest_hint and by_path_suffix:
for match_sha1 in by_path_suffix.get(dest_hint, []):
if match_sha1 in files_db:
path = files_db[match_sha1]["path"]
if os.path.exists(path):
if not has_strong_hash:
return path, "path_exact"
status = _record_match_status(match_sha1)
if status:
return path, status
# 3. No MD5 = any file with that name or alias (existence check)
def _size_ok(match_sha1: str) -> bool:
@@ -550,7 +636,7 @@ def resolve_local_file(
file_entry, files_db.get(match_sha1, {}).get("size")
)
if not md5_list:
if not has_strong_hash:
candidates = []
for try_name in names_to_try:
for match_sha1 in by_name.get(try_name, []):
@@ -575,7 +661,7 @@ def resolve_local_file(
candidates = [p for p in candidates if ".zip" in os.path.basename(p)]
primary = [p for p in candidates if "/.variants/" not in p]
if primary or candidates:
return (primary[0] if primary else candidates[0]), "exact"
return (primary[0] if primary else candidates[0]), "name_exact"
# 4. Name + alias fallback with md5_composite + direct MD5 per candidate
md5_set = set(md5_list)
@@ -595,17 +681,17 @@ def resolve_local_file(
candidates = [
(p, m) for p, m in candidates if ".zip" in os.path.basename(p)
]
if md5_set:
if md5_set and not (sha1_candidates or sha256_candidates or crc_raw):
for path, db_md5 in candidates:
if ".zip" in os.path.basename(path):
try:
composite = md5_composite(path).lower()
if composite in md5_set:
return path, "exact"
return path, "md5_composite_exact"
except (zipfile.BadZipFile, OSError):
pass
if db_md5.lower() in md5_set:
return path, "exact"
return path, "md5_exact"
# When zipped_file is set, only accept candidates that contain it
if zipped_file:
valid = []
@@ -628,7 +714,12 @@ def resolve_local_file(
# 5. zipped_file content match via pre-built index (last resort:
# matches inner ROM MD5 across ALL ZIPs in the repo, so only use
# when name-based resolution failed entirely)
if zipped_file and md5_list and zip_contents:
if (
zipped_file
and md5_list
and zip_contents
and not (sha1_candidates or sha256_candidates or crc_raw)
):
for md5_candidate in md5_list:
if md5_candidate in zip_contents:
zip_sha1 = zip_contents[md5_candidate]
@@ -638,7 +729,7 @@ def resolve_local_file(
return path, "zip_exact"
# MAME clone fallback: if a file was deduped, resolve via canonical
if _depth < 3:
if _depth < 3 and not has_strong_hash:
clone_map = _get_mame_clone_map()
canonical = clone_map.get(name)
if canonical and canonical != name:
@@ -655,6 +746,42 @@ def resolve_local_file(
return result[0], "mame_clone"
# Data directory fallback: scan data/ caches for matching filename
def _unindexed_path_status(candidate: str) -> str:
"""Validate a data-directory candidate without trusting its filename."""
if not has_strong_hash:
return "data_dir"
algorithms: set[str] = set()
if sha1_candidates:
algorithms.add("sha1")
if sha256_candidates:
algorithms.add("sha256")
if md5_list and not zipped_file:
algorithms.add("md5")
if crc_raw:
algorithms.add("crc32")
actual = compute_hashes(candidate, frozenset(algorithms)) if algorithms else {}
if sha1_candidates and actual.get("sha1", "").lower() not in sha1_candidates:
return "hash_mismatch"
if sha256_candidates and actual.get("sha256", "").lower() not in sha256_candidates:
return "hash_mismatch"
if md5_list and not zipped_file and not any(
actual.get("md5", "").lower().startswith(expected) for expected in md5_list
):
return "hash_mismatch"
if crc_raw and actual.get("crc32", "").lower() != crc_raw:
return "hash_mismatch"
if crc_raw and declared_size is not None:
allowed_sizes = declared_size if isinstance(declared_size, list) else [declared_size]
if os.path.getsize(candidate) not in allowed_sizes:
return "hash_mismatch"
if zipped_file and md5_list and not any(
check_inside_zip(candidate, zipped_file, expected) == "ok"
for expected in md5_list
):
return "hash_mismatch"
return "data_dir_hash_exact"
data_dir_mismatch: str | None = None
if data_dir_registry:
for _dd_key, dd_entry in data_dir_registry.items():
cache_dir = dd_entry.get("local_cache", "")
@@ -664,7 +791,10 @@ def resolve_local_file(
# Exact relative path
candidate = os.path.join(cache_dir, try_name)
if os.path.isfile(candidate):
return candidate, "data_dir"
status = _unindexed_path_status(candidate)
if status != "hash_mismatch":
return candidate, status
data_dir_mismatch = data_dir_mismatch or candidate
# Basename walk: find file anywhere in cache tree (case-insensitive)
basename_targets = {
(n.rsplit("/", 1)[-1] if "/" in n else n).casefold()
@@ -673,11 +803,18 @@ def resolve_local_file(
for root, _dirs, fnames in os.walk(cache_dir):
for fn in fnames:
if fn.casefold() in basename_targets:
return os.path.join(root, fn), "data_dir"
candidate = os.path.join(root, fn)
status = _unindexed_path_status(candidate)
if status != "hash_mismatch":
return candidate, status
data_dir_mismatch = data_dir_mismatch or candidate
if data_dir_mismatch:
return data_dir_mismatch, "hash_mismatch"
# Agnostic fallback: for filename-agnostic files, find any DB file
# matching the system path prefix and size criteria
if file_entry.get("agnostic"):
if file_entry.get("agnostic") and not has_strong_hash:
agnostic_prefix = file_entry.get("agnostic_path_prefix", "")
min_size = file_entry.get("min_size", 0)
max_size = file_entry.get("max_size", float("inf"))
@@ -1228,8 +1365,10 @@ def fetch_large_file(
dest_dir: str = LARGE_FILES_CACHE,
expected_sha1: str = "",
expected_md5: str = "",
*,
offline: bool = False,
) -> str | None:
"""Download a large file from the 'large-files' GitHub release if not cached."""
"""Return a verified cached large file, downloading it only when allowed."""
cached = os.path.join(dest_dir, name)
if os.path.exists(cached):
if expected_sha1 or expected_md5:
@@ -1249,6 +1388,9 @@ def fetch_large_file(
else:
return cached
if offline:
return None
os.makedirs(dest_dir, exist_ok=True)
# A per-process scratch name: two runs fetching the same asset into one
# shared path interleave their writes into a full-size, corrupt file.
@@ -1303,15 +1445,118 @@ def fetch_large_file(
return cached
def safe_extract_zip(zip_path: str, dest_dir: str) -> None:
"""Extract a ZIP file safely, preventing zip-slip path traversal."""
MAX_ZIP_MEMBERS = 100_000
MAX_ZIP_MEMBER_SIZE = 8 * 1024 * 1024 * 1024
# The largest generated pack is already ~5 GB uncompressed and the collection
# only grows; this bounds a malicious archive without capping a real one.
MAX_ZIP_TOTAL_SIZE = 64 * 1024 * 1024 * 1024
# DEFLATE cannot exceed roughly 1,032:1, so this rejects a declared ratio no
# real DEFLATE member can reach. Methods with a higher ceiling (bzip2, LZMA)
# are exempt and bounded by the per-member and per-archive size limits alone.
MAX_ZIP_COMPRESSION_RATIO = 1_100
_BOUNDED_RATIO_METHODS = (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED)
def safe_extract_zip(
zip_path: str,
dest_dir: str,
*,
max_members: int = MAX_ZIP_MEMBERS,
max_member_size: int = MAX_ZIP_MEMBER_SIZE,
max_total_size: int = MAX_ZIP_TOTAL_SIZE,
max_compression_ratio: int = MAX_ZIP_COMPRESSION_RATIO,
) -> None:
"""Extract a ZIP with traversal, link and resource-limit protection.
Files are streamed to a temporary sibling and atomically installed only
after their declared length and CRC have been checked by ``zipfile``.
"""
dest = os.path.realpath(dest_dir)
os.makedirs(dest, exist_ok=True)
with zipfile.ZipFile(zip_path, "r") as zf:
for member in zf.infolist():
member_path = os.path.realpath(os.path.join(dest, member.filename))
if not member_path.startswith(dest + os.sep) and member_path != dest:
raise ValueError(f"Zip slip detected: {member.filename}")
zf.extract(member, dest)
members = zf.infolist()
if len(members) > max_members:
raise ValueError(
f"ZIP has {len(members)} members; limit is {max_members}"
)
declared_total = 0
seen: set[str] = set()
for member in members:
# Archives written on Windows store a backslash separator. It is a
# separator, not a filename character, so it is normalized before
# the component checks rather than rejected.
name = member.filename.replace("\\", "/")
if not name or "\x00" in name:
raise ValueError(f"Unsafe ZIP member name: {member.filename!r}")
if name.startswith("/") or re.match(r"^[A-Za-z]:", name):
raise ValueError(f"Absolute ZIP member path: {name}")
parts = [part for part in name.split("/") if part]
if any(part in (".", "..") for part in parts):
raise ValueError(f"ZIP traversal detected: {name}")
normalized = "/".join(parts)
if normalized in seen:
raise ValueError(f"Duplicate ZIP member path: {name}")
seen.add(normalized)
mode = (member.external_attr >> 16) & 0xFFFF
file_type = stat.S_IFMT(mode)
if file_type not in (0, stat.S_IFREG, stat.S_IFDIR):
raise ValueError(f"ZIP link or special file rejected: {name}")
if member.flag_bits & 0x1:
raise ValueError(f"Encrypted ZIP member rejected: {name}")
if member.file_size > max_member_size:
raise ValueError(
f"ZIP member {name} is {member.file_size} bytes; "
f"limit is {max_member_size}"
)
declared_total += member.file_size
if declared_total > max_total_size:
raise ValueError(
f"ZIP expands to {declared_total} bytes; limit is {max_total_size}"
)
if member.file_size and member.compress_type in _BOUNDED_RATIO_METHODS:
if member.compress_size == 0:
raise ValueError(f"Invalid compression size for ZIP member: {name}")
if member.file_size / member.compress_size > max_compression_ratio:
raise ValueError(f"Suspicious compression ratio for ZIP member: {name}")
target = os.path.realpath(os.path.join(dest, *parts))
if not target.startswith(dest + os.sep) and target != dest:
raise ValueError(f"ZIP traversal detected: {name}")
if member.is_dir() or name.endswith("/"):
os.makedirs(target, exist_ok=True)
continue
os.makedirs(os.path.dirname(target), exist_ok=True)
tmp_path = ""
try:
with tempfile.NamedTemporaryFile(
mode="wb", dir=os.path.dirname(target), delete=False
) as tmp_file:
tmp_path = tmp_file.name
actual_size = 0
with zf.open(member, "r") as source:
while True:
chunk = source.read(1024 * 1024)
if not chunk:
break
actual_size += len(chunk)
if actual_size > member.file_size or actual_size > max_member_size:
raise ValueError(
f"ZIP member exceeded declared or configured size: {name}"
)
tmp_file.write(chunk)
if actual_size != member.file_size:
raise ValueError(
f"ZIP member size mismatch for {name}: "
f"{actual_size} != {member.file_size}"
)
os.replace(tmp_path, target)
tmp_path = ""
finally:
if tmp_path and os.path.exists(tmp_path):
os.unlink(tmp_path)
def list_emulator_profiles(emulators_dir: str, skip_aliases: bool = True) -> None:
scripts/cross_reference.py
+30 -11
View File
@@ -230,17 +230,26 @@ def cross_reference(
gaps = []
covered = []
unsourceable_list: list[dict] = []
archive_gaps: dict[str, dict] = {}
seen_files: set[str] = set()
archive_gaps: dict[tuple, dict] = {}
seen_files: set[tuple] = set()
for f in emu_files:
fname = f.get("name", "")
if not fname or fname in seen_files:
effective_path = f.get("path") or fname
seen_key = (
fname,
f.get("archive"),
effective_path,
f.get("system"),
f.get("variant_group"),
f.get("mode", "both"),
)
if not fname or seen_key in seen_files:
continue
# Collect unsourceable files separately (documented, not a gap)
unsourceable_reason = f.get("unsourceable", "")
if unsourceable_reason:
seen_files.add(fname)
seen_files.add(seen_key)
unsourceable_list.append({
"name": fname,
"required": f.get("required", False),
@@ -279,25 +288,33 @@ def cross_reference(
in_platform = archive in platform_names
if in_platform:
seen_files.add(fname)
seen_files.add(seen_key)
covered.append({
"name": fname,
"path": effective_path,
"required": f.get("required", False),
"in_platform": True,
})
continue
seen_files.add(fname)
seen_files.add(seen_key)
# Group archived files by archive name
if archive:
if archive not in archive_gaps:
archive_key = (
archive,
f.get("system"),
f.get("variant_group"),
f.get("mode", "both"),
)
if archive_key not in archive_gaps:
source = _resolve_archive_source(
archive, by_name, by_name_lower, data_names,
by_path_suffix,
)
archive_gaps[archive] = {
archive_gaps[archive_key] = {
"name": archive,
"path": archive,
"required": False,
"note": "",
"source_ref": "",
@@ -308,7 +325,7 @@ def cross_reference(
"archive_file_count": 0,
"archive_required_count": 0,
}
entry = archive_gaps[archive]
entry = archive_gaps[archive_key]
entry["archive_file_count"] += 1
if f.get("required", False):
entry["archive_required_count"] += 1
@@ -351,8 +368,9 @@ def cross_reference(
break
# Try SHA1 hash match
if source is None:
sha1 = f.get("sha1", "")
if sha1 and sha1 in db_files:
raw_sha1 = f.get("sha1", "")
sha1_values = raw_sha1 if isinstance(raw_sha1, list) else [raw_sha1]
if any(value and value in db_files for value in sha1_values):
source = "bios"
# Try CRC32 hash match
if source is None:
@@ -366,6 +384,7 @@ def cross_reference(
entry = {
"name": fname,
"path": effective_path,
"required": f.get("required", False),
"note": f.get("note", ""),
"source_ref": f.get("source_ref", ""),
scripts/generate_db.py
+1
View File
@@ -342,6 +342,7 @@ def main():
total_size = sum(entry["size"] for entry in files.values())
database = {
"schema_version": 1,
"generated_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"total_files": len(files),
"total_size": total_size,
scripts/generate_pack.py
+467 -138
View File
File diff suppressed because it is too large. Load diff
scripts/pipeline.py
+70 -11
View File
@@ -4,6 +4,7 @@
Steps:
1. generate_db.py --force (rebuild database.json from bios/)
1b. provenance_report.py (dump-catalog coverage from provenance/)
1c. romset_recipes.py (archive identification, reconstruction targets)
2. refresh_data_dirs.py (update Dolphin Sys, PPSSPP, etc.)
3. verify.py --all (check all platforms)
4. generate_pack.py --all (build ZIP packs)
@@ -24,6 +25,7 @@ Usage:
from __future__ import annotations
import argparse
import re
import subprocess
import sys
import time
@@ -57,8 +59,6 @@ def parse_verify_counts(output: str) -> dict[str, tuple[int, int]]:
Matches: "Label: X/Y OK ..." or "Label: X/Y present ..."
Returns {group_label: (ok, total)}.
"""
import re
counts = {}
for line in output.splitlines():
m = re.match(r"^(.+?):\s+(\d+)/(\d+)\s+(OK|present)", line)
@@ -75,14 +75,16 @@ def parse_pack_counts(output: str) -> dict[str, tuple[int, int]]:
Returns {pack_label: (ok, total)}.
"""
import re
counts = {}
current_label = ""
for line in output.splitlines():
m = re.match(r"Generating (?:shared )?pack for (.+)\.\.\.", line)
if m:
current_label = m.group(1)
# Labels carry execution metadata such as ``[source=full]``.
# It is not part of the platform identity used for consistency.
current_label = re.sub(
r"\s+\[source=[^\]]+\]$", "", m.group(1).strip()
)
continue
if "files packed" not in line:
continue
@@ -100,34 +102,77 @@ def parse_pack_counts(output: str) -> dict[str, tuple[int, int]]:
return counts
def parse_pack_exclusions(output: str) -> dict[str, int]:
"""Extract intentional unsafe-omission counts from pack output."""
exclusions: dict[str, int] = {}
current_label = ""
for line in output.splitlines():
label_match = re.match(r"Generating (?:shared )?pack for (.+)\.\.\.", line)
if label_match:
current_label = re.sub(
r"\s+\[source=[^\]]+\]$", "", label_match.group(1).strip()
)
continue
if "files packed" not in line:
continue
excluded_match = re.search(r"(\d+) unsafe excluded", line)
exclusions[current_label] = (
int(excluded_match.group(1)) if excluded_match else 0
)
return exclusions
def _match_key(label: str) -> set[str]:
"""Comparable identity for a platform label.
Display labels and registry ids differ in punctuation and spacing
(``MiSTer FPGA`` against ``misterfpga``), and grouped packs join their
members with either separator.
"""
return {
re.sub(r"[^a-z0-9]+", "", part.strip().lower())
for part in label.replace("+", "/").split("/")
} - {""}
def check_consistency(verify_output: str, pack_output: str) -> bool:
"""Verify that check counts match between verify and pack for each platform."""
v = parse_verify_counts(verify_output)
p = parse_pack_counts(pack_output)
excluded = parse_pack_exclusions(pack_output)
print("\n--- 5/8 consistency check ---")
all_ok = True
for v_label, (v_ok, v_total) in sorted(v.items()):
# Match by name overlap (handles "Lakka + RetroArch" vs "Lakka / RetroArch")
# Match by normalized name overlap. Platform display labels and
# registry IDs legitimately differ in punctuation and spacing
# (notably ``MiSTer FPGA`` vs ``misterfpga``).
p_match = None
v_names = _match_key(v_label)
for p_label in p:
v_names = {n.strip().lower() for n in v_label.split("/")}
p_names = {n.strip().lower() for n in p_label.replace("+", "/").split("/")}
if v_names & p_names:
if v_names & _match_key(p_label):
p_match = p_label
break
if p_match:
p_ok, p_total = p[p_match]
p_excluded = excluded.get(p_match, 0)
if v_total != p_total:
print(f" {v_label}: MISMATCH total verify {v_total} != pack {p_total}")
all_ok = False
elif p_ok < v_ok:
elif p_ok + p_excluded < v_ok:
print(
f" {v_label}: MISMATCH pack {p_ok} OK < verify {v_ok} OK (/{v_total})"
f" {v_label}: MISMATCH pack accounts for "
f"{p_ok} OK + {p_excluded} unsafe exclusions "
f"< verify {v_ok} OK (/{v_total})"
)
all_ok = False
elif p_ok < v_ok:
print(
f" {v_label}: verify {v_ok}/{v_total} native; pack {p_ok} safe, "
f"{p_excluded} unsafe excluded OK"
)
elif p_ok == v_ok:
print(
f" {v_label}: verify {v_ok}/{v_total} == pack {p_ok}/{p_total} OK"
@@ -138,6 +183,7 @@ def check_consistency(verify_output: str, pack_output: str) -> bool:
)
else:
print(f" {v_label}: {v_ok}/{v_total} (no separate pack)")
all_ok = False
status = "OK" if all_ok else "FAILED"
print(f"--- consistency check: {status} ---")
@@ -219,6 +265,16 @@ def main():
results["provenance"] = ok
all_ok = all_ok and ok
# Step 1c: Which emulator version each arcade archive corresponds to, and
# which pinned archive the collection could rebuild from ROMs it holds.
# Read-only: writing reconstructions is an explicit --write invocation.
ok, out = run(
[sys.executable, "scripts/romset_recipes.py"],
"1c romset recipes",
)
results["romset_recipes"] = ok
all_ok = all_ok and ok
# Step 2: Refresh data directories
if not args.offline:
ok, out = run(
@@ -226,6 +282,7 @@ def main():
"2/8 refresh data directories",
)
results["refresh_data"] = ok
all_ok = all_ok and ok
else:
print("\n--- 2/8 refresh data directories: SKIPPED (--offline) ---")
results["refresh_data"] = True
@@ -237,6 +294,7 @@ def main():
"2a refresh MAME hashes",
)
results["mame_hashes"] = ok
all_ok = all_ok and ok
else:
print("\n--- 2a refresh MAME hashes: SKIPPED (--offline) ---")
results["mame_hashes"] = True
@@ -248,6 +306,7 @@ def main():
"2a2 refresh FBNeo hashes",
)
results["fbneo_hashes"] = ok
all_ok = all_ok and ok
else:
print("\n--- 2a2 refresh FBNeo hashes: SKIPPED (--offline) ---")
results["fbneo_hashes"] = True
scripts/region.py
+47 -22
View File
@@ -128,9 +128,9 @@ def region_tag(requested: list[str]) -> str:
def build_region_index(profiles: dict) -> dict[str, dict]:
"""Build a region lookup from emulator profiles.
Keyed by the entry's path when present, by name otherwise. Entries sharing a
key have their regions unioned, so an ambiguous lookup keeps more files
rather than fewer.
Keyed by the entry's path when present, by name otherwise. Untagged
declarations are recorded as ambiguity evidence: if the same lookup key is
both tagged and untagged, filtering keeps it instead of inventing a region.
"""
index: dict[str, dict] = {}
for emu_name, profile in sorted(profiles.items()):
@@ -145,16 +145,19 @@ def build_region_index(profiles: dict) -> dict[str, dict]:
raise ValueError(
f"{emu_name}: {f.get('name', '?')}: {exc}"
) from exc
if not regions:
continue
name = f.get("name", "")
path = f.get("path") or ""
# Path-keyed so same-named entries stay separate (Dolphin declares
# three IPL.bin), name-keyed so a candidate identified by name alone
# sees the union and is never dropped on ambiguity.
for key in {path, name} - {""}:
entry = index.setdefault(key, {"regions": set(), "emulators": []})
entry = index.setdefault(
key,
{"regions": set(), "has_untagged": False, "emulators": []},
)
entry["regions"] |= regions
if not regions:
entry["has_untagged"] = True
if emu_name not in entry["emulators"]:
entry["emulators"].append(emu_name)
return index
@@ -170,14 +173,16 @@ def lookup_regions(index: dict[str, dict], destination: str, name: str) -> set[s
if destination:
entry = index.get(destination)
if entry:
return set(entry["regions"])
return set() if entry.get("has_untagged") else set(entry["regions"])
parts = destination.split("/")
for i in range(1, len(parts)):
entry = index.get("/".join(parts[i:]))
if entry:
return set(entry["regions"])
return set() if entry.get("has_untagged") else set(entry["regions"])
entry = index.get(name)
return set(entry["regions"]) if entry else set()
if not entry or entry.get("has_untagged"):
return set()
return set(entry["regions"])
def _competing_ranks(
@@ -206,8 +211,10 @@ def resolve_region_drops(
) -> set[str]:
"""Destinations to skip for a requested region priority list.
Per group, only the best rank actually present survives. A destination kept
by any group is kept overall.
Per group, an exact/parent regional match beats other regional candidates.
A world candidate beats unmatched regional fallbacks, while untagged files
always survive. If neither a requested nor a world candidate exists, all
regional candidates survive so filtering can never empty a group.
"""
if not requested:
return set()
@@ -215,17 +222,31 @@ def resolve_region_drops(
keep: set[str] = set()
drop: set[str] = set()
for members in groups.values():
ranked = _competing_ranks(members, index, requested)
competing = {dest for _r, dest in ranked}
keep |= {dest for dest, _name in members if dest not in competing}
if not ranked:
continue
best = min(r for r, _ in ranked)
for r, destination in ranked:
if r == best:
keep.add(destination)
regional: list[tuple[int, str]] = []
world: set[str] = set()
untagged: set[str] = set()
for destination, name in members:
regions = lookup_regions(index, destination, name)
if not regions:
untagged.add(destination)
elif WORLD in regions:
world.add(destination)
else:
drop.add(destination)
regional.append((rank(regions, requested), destination))
keep |= untagged | world
if not regional:
continue
matched = [(r, destination) for r, destination in regional if r < len(requested)]
if matched:
best = min(r for r, _destination in matched)
keep |= {destination for r, destination in matched if r == best}
drop |= {destination for _r, destination in regional if destination not in keep}
elif world:
drop |= {destination for _r, destination in regional}
else:
# Preserve every unmatched candidate as a visible fallback.
keep |= {destination for _r, destination in regional}
return drop - keep
@@ -240,6 +261,10 @@ def fallback_groups(
out: list[str] = []
for group_id, members in groups.items():
ranked = _competing_ranks(members, index, requested)
if ranked and min(r for r, _ in ranked) == len(requested):
has_world = any(
WORLD in lookup_regions(index, destination, name)
for destination, name in members
)
if ranked and not has_world and min(r for r, _ in ranked) == len(requested):
out.append(group_id)
return sorted(out)
scripts/region_audit.py
+11 -7
View File
@@ -19,7 +19,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import region
from common import load_emulator_profiles, load_database
from common import load_emulator_profiles, load_database, parse_md5_list
# No-Intro filename tokens: full English territory names.
NOINTRO = {
@@ -104,12 +104,16 @@ def resolve_sha1(file_entry: dict, db: dict) -> str | None:
"""Resolve a profile file entry to a repo SHA1, or None when ambiguous."""
files = db["files"]
indexes = db["indexes"]
declared = str(file_entry.get("sha1") or "").lower()
if declared and declared in files:
return declared
md5 = str(file_entry.get("md5") or "").lower()
if md5 and md5 in indexes["by_md5"]:
return indexes["by_md5"][md5]
raw_sha1 = file_entry.get("sha1")
declared = raw_sha1 if isinstance(raw_sha1, list) else [raw_sha1]
sha1_hits = [str(value).lower() for value in declared if value]
sha1_hits = [value for value in sha1_hits if value in files]
if len(sha1_hits) == 1:
return sha1_hits[0]
for md5 in parse_md5_list(file_entry.get("md5")):
hit = indexes["by_md5"].get(md5)
if hit:
return hit
hits = indexes["by_name"].get(file_entry.get("name", ""), [])
if isinstance(hits, str):
hits = [hits]
scripts/verify.py
+46 -9
View File
@@ -370,9 +370,9 @@ def find_undeclared_files(
relevant = resolve_platform_cores(config, profiles, target_cores=target_cores)
standalone_set = set(str(c) for c in config.get("standalone_cores", []))
undeclared = []
seen_files: set[tuple[str, str | None]] = set()
seen_files: set[tuple] = set()
# Track archives: archive_name -> {in_repo, emulator, files: [...], ...}
archive_entries: dict[str, dict] = {}
archive_entries: dict[tuple, dict] = {}
for emu_name, profile in sorted(profiles.items()):
if profile.get("type") in ("launcher", "alias"):
@@ -391,9 +391,25 @@ def find_undeclared_files(
for f in profile.get("files", []):
fname = f.get("name", "")
# Dedup by (name, archive): a filename declared loose by one core
# and inside an archive by another are distinct packaging shapes
seen_key = (fname, f.get("archive"))
effective_path = (
f.get("standalone_path") if is_standalone else f.get("path")
) or fname
raw_regions = f.get("region") or []
region_key = tuple(
str(value) for value in (
raw_regions if isinstance(raw_regions, list) else [raw_regions]
)
)
# Same-name requirements at different paths, for different systems,
# or in distinct variant groups are not interchangeable.
seen_key = (
fname,
f.get("archive"),
effective_path,
f.get("system"),
f.get("variant_group"),
region_key,
)
if not fname or seen_key in seen_files:
continue
# Skip unsourceable files (documented reason, not a gap)
@@ -436,13 +452,25 @@ def find_undeclared_files(
# Archived files are grouped by archive
if archive:
if archive not in archive_entries:
archive_key = (
archive,
f.get("system"),
f.get("variant_group"),
region_key,
is_standalone,
)
if archive_key not in archive_entries:
in_repo = _name_in_index(
archive, by_name, by_path_suffix, data_names,
by_name_lower,
)
archive_entries[archive] = {
archive_entries[archive_key] = {
"profile": emu_name,
"emulator": profile.get("emulator", emu_name),
"systems": list(profile.get("systems", [])),
"system": f.get("system"),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"name": archive,
"archive": archive,
"path": archive,
@@ -457,7 +485,7 @@ def find_undeclared_files(
"archive_file_count": 0,
"archive_required_count": 0,
}
entry = archive_entries[archive]
entry = archive_entries[archive_key]
entry["archive_file_count"] += 1
if f.get("required", False):
entry["archive_required_count"] += 1
@@ -487,13 +515,22 @@ def find_undeclared_files(
if not in_repo:
# Hash fallback: the repo may hold the content under a
# different filename (exos21.rom vs exos21.bin)
# generate_pack ships a core extra whose local copy
# contradicts the declared hash and reports the divergence.
# verify must agree with the builder or the two reports
# disagree on the same file.
_lp, _st = resolve_local_file(f, db, dest_hint=dest)
in_repo = _st not in ("not_found",) and _lp is not None
in_repo = _st != "not_found" and _lp is not None
checks = _parse_validation(f.get("validation"))
undeclared.append(
{
"profile": emu_name,
"emulator": profile.get("emulator", emu_name),
"systems": list(profile.get("systems", [])),
"system": f.get("system"),
"region": f.get("region"),
"variant_group": f.get("variant_group"),
"name": fname,
"path": dest,
"required": f.get("required", False),