feat: run content checks profiles declare

validation: [romdir] walks a PS2 ROM directory the way the profile's
fork reads it, masked_sha1 hashes a file with a byte range cleared, and
size_multiple bounds a size to a granule. The resolver name steps, the
pack and manifest scans, the emulator expansion and verify call the same
content_fits. A PCSX2 companion is taken only beside an image that
passes, and a companion placeholder no longer seeds a scan of its own.
This commit is contained in:
Abdessamad Derraz committed 2026-11-10 12:52:28 +01:00
1 parent 51e5fd0dbd
commit ec9fe03673
12 files changed
+777 -29

No files matched your search

+124 -11
View File
@@ -316,11 +316,110 @@ def adler32_fits(file_entry: dict, path: str, db: dict | None = None) -> bool:
return chosen is None or actual == chosen
# The content tests a profile can declare in ``validation:``, each run by
# reading the candidate the way the emulator does.
CONTENT_CHECKS = frozenset({"romdir", "masked_sha1"})
def _core_checks(file_entry: dict) -> list:
validation = file_entry.get("validation")
if isinstance(validation, dict):
validation = validation.get("core", [])
return list(validation or [])
def declared_content_checks(file_entry: dict) -> set[str]:
"""The content tests the entry's code runs on a candidate."""
return set(CONTENT_CHECKS.intersection(_core_checks(file_entry)))
@functools.lru_cache(maxsize=4096)
def _romdir_verdict(path: str, size: int, mtime_ns: int, spec: str) -> bool:
from romdir import romdir_accepts # noqa: PLC0415
with open(path, "rb") as f:
data = f.read()
return romdir_accepts(data, json.loads(spec))
@functools.lru_cache(maxsize=4096)
def _masked_sha1_verdict(path: str, size: int, mtime_ns: int, spec: str) -> bool:
"""Amiberry's flash image identification (getromdatabydata, rommgr.cpp).
A CyberStorm or Blizzard PPC image carries its board configuration in
bytes 0x10-0x2f, so the table SHA-1 is taken with those bytes cleared,
over the first 0x1c000 bytes, and for a 512 KB file also over the
first 0x50000. ``reads`` lists the file sizes the code tries the
transform on and the lengths it hashes for each.
"""
rules = json.loads(spec)
reads = {int(r["size"]): [int(n) for n in r["lengths"]] for r in rules.get("reads", [])}
if size not in reads:
return False
with open(path, "rb") as f:
data = bytearray(f.read())
start, end = (int(v) for v in rules.get("zeroed", [0, 0]))
data[start:end] = bytes(max(0, min(end, len(data)) - start))
wanted = str(rules.get("sha1", "")).lower()
return any(hashlib.sha1(data[:length]).hexdigest() == wanted for length in reads[size])
def content_fits(file_entry: dict, path: str) -> bool:
"""Whether a candidate passes the content tests its entry declares.
The PCSX2 family lists any file of its BIOS folder whose ROM directory
holds RESET and ROMVER (LoadBiosVersion, pcsx2/ps2/BiosTools.cpp), with
no name and, in most forks, no size: ``validation: [romdir]`` and the
``romdir:`` spec of how that fork reads the table (romdir.py). Amiberry
names a flash image by the SHA-1 of its bytes with the configuration
cleared (``masked_sha1``). A file that cannot be read fails. An entry
declaring no content test accepts every candidate.
"""
checks = declared_content_checks(file_entry)
if not checks:
return True
try:
stat = os.stat(path)
except OSError:
return False
verdicts = {"romdir": _romdir_verdict, "masked_sha1": _masked_sha1_verdict}
for check in sorted(checks):
spec = json.dumps(file_entry.get(check) or {}, sort_keys=True)
try:
if not verdicts[check](path, stat.st_size, stat.st_mtime_ns, spec):
return False
except OSError:
return False
return True
def shape_fits(file_entry: dict, path: str, size: int | None = None) -> bool:
"""Size and content together: what a scan by shape checks per candidate."""
if size is None:
try:
size = os.path.getsize(path)
except OSError:
return False
return size_fits(file_entry, size) and content_fits(file_entry, path)
def declares_shape(file_entry: dict) -> bool:
"""Whether the entry says anything that tells one candidate from another."""
return bool(
file_entry.get("size") or file_entry.get("min_size")
or file_entry.get("max_size") or file_entry.get("size_multiple")
or declared_content_checks(file_entry)
)
def size_fits(file_entry: dict, candidate_size: int) -> bool:
"""Whether a size is one the entry declares.
``size`` is an exact size or the list of sizes the code accepts, and it
outranks a ``min_size``/``max_size`` range. Nothing declared fits all.
outranks a ``min_size``/``max_size`` range. ``size_multiple`` is a
granule the size must be a whole number of, as xemu maps any flash
image whose size is a positive multiple of 64 KiB (hw/xbox/xbox.c).
Nothing declared fits all.
"""
declared = file_entry.get("size")
if declared is not None:
@@ -330,7 +429,10 @@ def size_fits(file_entry: dict, candidate_size: int) -> bool:
high = file_entry.get("max_size")
if low is not None and candidate_size < low:
return False
return not (high is not None and candidate_size > high)
if high is not None and candidate_size > high:
return False
multiple = file_entry.get("size_multiple")
return not (multiple and candidate_size % int(multiple))
@@ -993,6 +1095,7 @@ def declared_hash_verdict(
if file_entry is not None and not (
name_match_size_ok(file_entry, os.path.getsize(candidate))
and adler32_fits(file_entry, candidate, db)
and content_fits(file_entry, candidate)
):
return "hash_mismatch"
return "data_dir"
@@ -1100,7 +1203,9 @@ def _data_dir_tail(
for candidate in hits:
if name_match_size_ok(
file_entry, os.path.getsize(candidate)
) and adler32_fits(file_entry, candidate, db):
) and adler32_fits(file_entry, candidate, db) and content_fits(
file_entry, candidate
):
return candidate
return None
@@ -1156,7 +1261,7 @@ def _resolve_agnostic(
path = entry.get("path", "")
if not path.startswith(prefix):
continue
if size_fits(file_entry, entry.get("size", 0)) and os.path.exists(path):
if os.path.exists(path) and shape_fits(file_entry, path, entry.get("size", 0)):
return path, "agnostic_fallback"
return None
@@ -1465,12 +1570,12 @@ def _by_affinity(paths: list[str], file_entry: dict, dest_hint: str) -> list[str
hint = dest_hint or file_entry.get("path") or file_entry.get("destination") or ""
wanted = _affinity_tokens(hint.rsplit("/", 1)[0] if "/" in hint else "")
owners = _owner_tokens(file_entry)
sized = any(file_entry.get(k) for k in ("size", "min_size", "max_size"))
sized = declares_shape(file_entry)
def score(path: str) -> tuple[int, bool, bool, bool, int]:
fits = 0
if sized and os.path.exists(path):
fits = int(size_fits(file_entry, os.path.getsize(path)))
fits = int(shape_fits(file_entry, path))
tokens = _affinity_tokens(path.rsplit("/", 1)[0])
# The owner's own copy ranks above any shared tree segment: NetherSX2's
# shaders/common/ beat armsx2's own fxaa.fx on two directory tokens.
@@ -1797,7 +1902,9 @@ def resolve_local_file(
# cpc6128.rom is not cap32's 32 KB one at the same tail.
if not name_match_size_ok(
file_entry, files_db[match_sha1].get("size")
) or not adler32_fits(file_entry, path, db):
) or not adler32_fits(file_entry, path, db) or not content_fits(
file_entry, path
):
continue
return path, "path_exact"
status = _record_match_status(match_sha1)
@@ -1830,7 +1937,7 @@ def resolve_local_file(
record = files_db.get(match_sha1, {})
return name_match_size_ok(file_entry, record.get("size")) and adler32_fits(
file_entry, record.get("path", ""), db
)
) and content_fits(file_entry, record.get("path", ""))
# An entry the profile documents as unsourceable is one the collection
# cannot hold: a font that only exists inside a paid package, cheat codes
@@ -2030,7 +2137,10 @@ def resolve_local_file(
# both an Apple IIgs image and a Macintosh one. It ranks rather than
# excludes, because a present file of the wrong size is reported as
# present and untested, and dropping it would report it as absent.
sized = [c for c in named if name_match_size_ok(file_entry, c[2])]
sized = [
c for c in named
if name_match_size_ok(file_entry, c[2]) and content_fits(file_entry, c[0])
]
if sized:
named = sized
@@ -2107,7 +2217,10 @@ def resolve_local_file(
if canonical and canonical != name:
canonical_entry = {
field: file_entry[field]
for field in ("size", "min_size", "max_size", "validation")
for field in (
"size", "min_size", "max_size", "size_multiple", "validation",
*sorted(CONTENT_CHECKS),
)
if field in file_entry
}
canonical_entry["name"] = canonical
@@ -3142,7 +3255,7 @@ def platform_layouts(
# validation on the way and resolved to an IBM PCjr ROM of the same name.
PROFILE_IDENTITY_FIELDS = (
"sha1", "md5", "sha256", "crc32", "size", "min_size", "max_size",
"validation", "aliases", "unsourceable",
"size_multiple", "validation", *sorted(CONTENT_CHECKS), "aliases", "unsourceable",
)