diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index bcb6703bf..7de174331 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -752,6 +752,11 @@ "max_size": { "type": "integer" }, + "size_multiple": { + "description": "Granule the file size must be a whole number of, the bounds still applying.", + "type": "integer", + "minimum": 1 + }, "md5": { "oneOf": [ { @@ -827,6 +832,35 @@ } ] }, + "masked_sha1": { + "description": "SHA-1 the code takes over a file with a byte range cleared (validation: masked_sha1), for each file size it tries.", + "type": "object", + "additionalProperties": false, + "properties": { + "zeroed": {"description": "Start and end offsets of the bytes cleared before hashing.", "type": "array", "items": {"type": "integer", "minimum": 0}, "minItems": 2, "maxItems": 2}, + "reads": {"type": "array", "minItems": 1, "items": {"type": "object", "additionalProperties": false, "properties": {"size": {"type": "integer"}, "lengths": {"type": "array", "items": {"type": "integer"}, "minItems": 1}}, "required": ["size", "lengths"]}}, + "sha1": {"type": "string", "pattern": "^[0-9a-f]{40}$"} + }, + "required": ["zeroed", "reads", "sha1"] + }, + "romdir": { + "description": "How the code reads a PS2 ROM directory (validation: romdir): the RESET search, the walk to ROMVER, and the serial and zone tests after it.", + "type": "object", + "additionalProperties": false, + "properties": { + "reset_compare": {"description": "Bytes of the entry name strncmp compares with RESET: 5 takes any name starting RESET, 10 wants RESET and a NUL.", "enum": [5, 10]}, + "missing_reset": {"description": "What 512 Ki entries without RESET do.", "enum": ["rejects", "ends_walk"]}, + "terminated_names": {"description": "The walk stops at a name with no NUL in its 10 bytes.", "type": "boolean"}, + "extinfo": {"description": "EXTINFO is read 0x10 past its offset, and an unreadable one ends the walk.", "type": "boolean"}, + "unreadable_romver": {"description": "What ROMVER bytes past the end of the file do.", "enum": ["rejects", "ends_walk", "ignored"]}, + "truncated_walk": {"description": "What a next entry past the end of the file does.", "enum": ["rejects", "ends_walk"]}, + "offset_width": {"description": "Bit width of the file offset sum.", "enum": [32, 64]}, + "refused_serials": {"description": "EXTINFO serials the code turns away.", "type": "array", "items": {"type": "string"}}, + "zones": {"description": "ROMVER zone codes the scan keeps, ROMVER[4] then [5] where the code reads it.", "type": "array", "items": {"type": "string", "minLength": 1, "maxLength": 2}}, + "refused_zones": {"description": "ROMVER zone codes the scan passes over.", "type": "array", "items": {"type": "string", "minLength": 1, "maxLength": 2}} + }, + "required": ["reset_compare", "missing_reset", "terminated_names", "extinfo", "unreadable_romver", "truncated_walk"] + }, "validation": { "description": "Checks the code performs. A dict splits core from upstream when they diverge.", "oneOf": [ diff --git a/scripts/common.py b/scripts/common.py index 7fc2bdf04..e3ee2b387 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -316,11 +316,110 @@ def adler32_fits(file_entry: dict, path: str, db: dict | None = None) -> bool: return chosen is None or actual == chosen +# The content tests a profile can declare in ``validation:``, each run by +# reading the candidate the way the emulator does. +CONTENT_CHECKS = frozenset({"romdir", "masked_sha1"}) + + +def _core_checks(file_entry: dict) -> list: + validation = file_entry.get("validation") + if isinstance(validation, dict): + validation = validation.get("core", []) + return list(validation or []) + + +def declared_content_checks(file_entry: dict) -> set[str]: + """The content tests the entry's code runs on a candidate.""" + return set(CONTENT_CHECKS.intersection(_core_checks(file_entry))) + + +@functools.lru_cache(maxsize=4096) +def _romdir_verdict(path: str, size: int, mtime_ns: int, spec: str) -> bool: + from romdir import romdir_accepts # noqa: PLC0415 + + with open(path, "rb") as f: + data = f.read() + return romdir_accepts(data, json.loads(spec)) + + +@functools.lru_cache(maxsize=4096) +def _masked_sha1_verdict(path: str, size: int, mtime_ns: int, spec: str) -> bool: + """Amiberry's flash image identification (getromdatabydata, rommgr.cpp). + + A CyberStorm or Blizzard PPC image carries its board configuration in + bytes 0x10-0x2f, so the table SHA-1 is taken with those bytes cleared, + over the first 0x1c000 bytes, and for a 512 KB file also over the + first 0x50000. ``reads`` lists the file sizes the code tries the + transform on and the lengths it hashes for each. + """ + rules = json.loads(spec) + reads = {int(r["size"]): [int(n) for n in r["lengths"]] for r in rules.get("reads", [])} + if size not in reads: + return False + with open(path, "rb") as f: + data = bytearray(f.read()) + start, end = (int(v) for v in rules.get("zeroed", [0, 0])) + data[start:end] = bytes(max(0, min(end, len(data)) - start)) + wanted = str(rules.get("sha1", "")).lower() + return any(hashlib.sha1(data[:length]).hexdigest() == wanted for length in reads[size]) + + +def content_fits(file_entry: dict, path: str) -> bool: + """Whether a candidate passes the content tests its entry declares. + + The PCSX2 family lists any file of its BIOS folder whose ROM directory + holds RESET and ROMVER (LoadBiosVersion, pcsx2/ps2/BiosTools.cpp), with + no name and, in most forks, no size: ``validation: [romdir]`` and the + ``romdir:`` spec of how that fork reads the table (romdir.py). Amiberry + names a flash image by the SHA-1 of its bytes with the configuration + cleared (``masked_sha1``). A file that cannot be read fails. An entry + declaring no content test accepts every candidate. + """ + checks = declared_content_checks(file_entry) + if not checks: + return True + try: + stat = os.stat(path) + except OSError: + return False + verdicts = {"romdir": _romdir_verdict, "masked_sha1": _masked_sha1_verdict} + for check in sorted(checks): + spec = json.dumps(file_entry.get(check) or {}, sort_keys=True) + try: + if not verdicts[check](path, stat.st_size, stat.st_mtime_ns, spec): + return False + except OSError: + return False + return True + + +def shape_fits(file_entry: dict, path: str, size: int | None = None) -> bool: + """Size and content together: what a scan by shape checks per candidate.""" + if size is None: + try: + size = os.path.getsize(path) + except OSError: + return False + return size_fits(file_entry, size) and content_fits(file_entry, path) + + +def declares_shape(file_entry: dict) -> bool: + """Whether the entry says anything that tells one candidate from another.""" + return bool( + file_entry.get("size") or file_entry.get("min_size") + or file_entry.get("max_size") or file_entry.get("size_multiple") + or declared_content_checks(file_entry) + ) + + def size_fits(file_entry: dict, candidate_size: int) -> bool: """Whether a size is one the entry declares. ``size`` is an exact size or the list of sizes the code accepts, and it - outranks a ``min_size``/``max_size`` range. Nothing declared fits all. + outranks a ``min_size``/``max_size`` range. ``size_multiple`` is a + granule the size must be a whole number of, as xemu maps any flash + image whose size is a positive multiple of 64 KiB (hw/xbox/xbox.c). + Nothing declared fits all. """ declared = file_entry.get("size") if declared is not None: @@ -330,7 +429,10 @@ def size_fits(file_entry: dict, candidate_size: int) -> bool: high = file_entry.get("max_size") if low is not None and candidate_size < low: return False - return not (high is not None and candidate_size > high) + if high is not None and candidate_size > high: + return False + multiple = file_entry.get("size_multiple") + return not (multiple and candidate_size % int(multiple)) @@ -993,6 +1095,7 @@ def declared_hash_verdict( if file_entry is not None and not ( name_match_size_ok(file_entry, os.path.getsize(candidate)) and adler32_fits(file_entry, candidate, db) + and content_fits(file_entry, candidate) ): return "hash_mismatch" return "data_dir" @@ -1100,7 +1203,9 @@ def _data_dir_tail( for candidate in hits: if name_match_size_ok( file_entry, os.path.getsize(candidate) - ) and adler32_fits(file_entry, candidate, db): + ) and adler32_fits(file_entry, candidate, db) and content_fits( + file_entry, candidate + ): return candidate return None @@ -1156,7 +1261,7 @@ def _resolve_agnostic( path = entry.get("path", "") if not path.startswith(prefix): continue - if size_fits(file_entry, entry.get("size", 0)) and os.path.exists(path): + if os.path.exists(path) and shape_fits(file_entry, path, entry.get("size", 0)): return path, "agnostic_fallback" return None @@ -1465,12 +1570,12 @@ def _by_affinity(paths: list[str], file_entry: dict, dest_hint: str) -> list[str hint = dest_hint or file_entry.get("path") or file_entry.get("destination") or "" wanted = _affinity_tokens(hint.rsplit("/", 1)[0] if "/" in hint else "") owners = _owner_tokens(file_entry) - sized = any(file_entry.get(k) for k in ("size", "min_size", "max_size")) + sized = declares_shape(file_entry) def score(path: str) -> tuple[int, bool, bool, bool, int]: fits = 0 if sized and os.path.exists(path): - fits = int(size_fits(file_entry, os.path.getsize(path))) + fits = int(shape_fits(file_entry, path)) tokens = _affinity_tokens(path.rsplit("/", 1)[0]) # The owner's own copy ranks above any shared tree segment: NetherSX2's # shaders/common/ beat armsx2's own fxaa.fx on two directory tokens. @@ -1797,7 +1902,9 @@ def resolve_local_file( # cpc6128.rom is not cap32's 32 KB one at the same tail. if not name_match_size_ok( file_entry, files_db[match_sha1].get("size") - ) or not adler32_fits(file_entry, path, db): + ) or not adler32_fits(file_entry, path, db) or not content_fits( + file_entry, path + ): continue return path, "path_exact" status = _record_match_status(match_sha1) @@ -1830,7 +1937,7 @@ def resolve_local_file( record = files_db.get(match_sha1, {}) return name_match_size_ok(file_entry, record.get("size")) and adler32_fits( file_entry, record.get("path", ""), db - ) + ) and content_fits(file_entry, record.get("path", "")) # An entry the profile documents as unsourceable is one the collection # cannot hold: a font that only exists inside a paid package, cheat codes @@ -2030,7 +2137,10 @@ def resolve_local_file( # both an Apple IIgs image and a Macintosh one. It ranks rather than # excludes, because a present file of the wrong size is reported as # present and untested, and dropping it would report it as absent. - sized = [c for c in named if name_match_size_ok(file_entry, c[2])] + sized = [ + c for c in named + if name_match_size_ok(file_entry, c[2]) and content_fits(file_entry, c[0]) + ] if sized: named = sized @@ -2107,7 +2217,10 @@ def resolve_local_file( if canonical and canonical != name: canonical_entry = { field: file_entry[field] - for field in ("size", "min_size", "max_size", "validation") + for field in ( + "size", "min_size", "max_size", "size_multiple", "validation", + *sorted(CONTENT_CHECKS), + ) if field in file_entry } canonical_entry["name"] = canonical @@ -3142,7 +3255,7 @@ def platform_layouts( # validation on the way and resolved to an IBM PCjr ROM of the same name. PROFILE_IDENTITY_FIELDS = ( "sha1", "md5", "sha256", "crc32", "size", "min_size", "max_size", - "validation", "aliases", "unsourceable", + "size_multiple", "validation", *sorted(CONTENT_CHECKS), "aliases", "unsourceable", ) diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 6a82613f9..2f993d8c8 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -42,6 +42,7 @@ from common import ( load_database, load_emulator_profiles, load_platform_config, + content_fits, name_match_size_ok, parse_md5_list, require_platform, @@ -247,6 +248,7 @@ def _resolve_source( sha1s = [sha1s] return any( name_match_size_ok(file_entry, db_files.get(sha1, {}).get("size")) + and content_fits(file_entry, db_files.get(sha1, {}).get("path", "")) and _record_carries(sha1, db_files.get(sha1, {}), declared) for sha1 in sha1s ) diff --git a/scripts/packextras.py b/scripts/packextras.py index 3731ec9ff..fa322051c 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -19,7 +19,7 @@ from common import resolve_platform_cores from common import core_system_dirs, platform_layouts, system_dir_path from common import standalone_set as platform_standalone_set from common import OUTER_ROOTS, outer_layout, sanitize_pack_path -from common import size_fits +from common import declares_shape, shape_fits, size_fits from validation import deliverable, read_from_save_dir, read_from_system_dir from packpaths import libretro_systems, save_destination, save_layout from region import FILE_GROUP @@ -602,7 +602,7 @@ def _agnostic_scan_extras( # the flag icons of an Android package walked into a PS2 pack. if f.get("category", "bios") != "bios": continue - if not (f.get("size") or f.get("min_size") or f.get("max_size")): + if not declares_shape(f): continue # The free name is the BIOS directory's: a shader, a sound or a # font the emulator reads from its resources tree has a fixed @@ -663,7 +663,7 @@ def _agnostic_scan_extras( continue elif not path.startswith(path_prefix): continue - if not companion_ext and not size_fits(f, entry.get("size", 0)): + if not companion_ext and not shape_fits(f, path, entry.get("size", 0)): continue scan_name = entry.get("name", "") if not scan_name: @@ -672,6 +672,12 @@ def _agnostic_scan_extras( stem, _, ext = scan_name.rpartition(".") if not stem or ext.lower() != companion_ext.lower(): continue + # Read only beside the image the core selected, so a + # companion counts only next to an image the image's + # own test accepts: Namco System 246 .nvm and .mec + # dumps sit beside no PS2 BIOS. + if not _image_beside(path, stem, image, files_db): + continue scan_name = f"{stem}.{companion_ext}" # Into the directory the seed entry names, not the root: # yaps2 reads its BIOS from pcsx2/bios/. @@ -745,7 +751,8 @@ def placeholder_directories( if f.get("size") and not size_fits(seed, f["size"]): continue local, status = resolve_local_file({**f, "source_profile": other_name}, db) - if local and "/" in local and resolution_is_hash_exact(status): + if (local and "/" in local and resolution_is_hash_exact(status) + and shape_fits(seed, local)): directories.add(local.rsplit("/", 1)[0]) return frozenset(directories) @@ -768,7 +775,11 @@ def _placeholder_scan( name = str(seed.get("name", "")) if "<" not in name or seed.get("category", "bios") != "bios": continue - if not (seed.get("size") or seed.get("min_size") or seed.get("max_size")): + if not declares_shape(seed): + continue + # .nvm is read beside the image the core selected, never on + # its own: its files come with each scanned image (_companions). + if _companion_of(profile, name)[1] is not None: continue directories = placeholder_directories( seed, emu_name, profile, db, @@ -780,7 +791,7 @@ def _placeholder_scan( path = entry.get("path", "") if "/" not in path or path.rsplit("/", 1)[0] not in directories: continue - if "/.variants/" in path or not size_fits(seed, entry.get("size", 0)): + if "/.variants/" in path or not shape_fits(seed, path, entry.get("size", 0)): continue scan_name = entry.get("name", "") dest = f"{folder}/{scan_name}" if folder else scan_name @@ -788,6 +799,7 @@ def _placeholder_scan( "name": scan_name, "destination": dest, "sha1": sha1, + "source_name": name, "source_system": seed.get("system"), "region": seed.get("region"), }) @@ -821,16 +833,34 @@ def expand_agnostic_entries( db.get("indexes", {}).get("by_name", {}), set(), "", {emu_name} if standalone else set(), ) - # A placeholder is answered by files found in its own folder: PCEE2's - # eeprom.dat, scanned at the root, said nothing about pcsx2/bios/. - covered = {e["destination"].rsplit("/", 1)[0] if "/" in e["destination"] else "" for e in scanned} + + # A placeholder is answered by the files its own scan found in its own + # folder: PCEE2's eeprom.dat, scanned at the root, said nothing about + # pcsx2/bios/, and its COH-H images, scanned into pcsx2/bios/, say + # nothing about the console .bin beside them. + def answered(extras: list[dict]) -> set[tuple[str, str]]: + return { + (str(e.get("source_name", "")), + e["destination"].rsplit("/", 1)[0] if "/" in e["destination"] else "") + for e in extras + } + + def is_answered(f: dict, covered: set[tuple[str, str]]) -> bool: + # A companion placeholder (.rom1) is read beside its image, so + # it is answered wherever the image placeholder is. + _ext, image = _companion_of(profile, str(f.get("name", ""))) + names = {f.get("name", "")} | ({image.get("name", "")} if image else set()) + return any((n, folder(f)) in covered for n in names) + + covered = answered(scanned) open_seeds = [ - f for f in files if "<" in str(f.get("name", "")) and folder(f) not in covered + f for f in files + if "<" in str(f.get("name", "")) and not is_answered(f, covered) ] scanned += _placeholder_scan( open_seeds, emu_name, profile, db, standalone, all_profiles ) - covered = {e["destination"].rsplit("/", 1)[0] if "/" in e["destination"] else "" for e in scanned} + covered = answered(scanned) if not scanned: return files concrete = [ @@ -848,7 +878,7 @@ def expand_agnostic_entries( ] return [ f for f in files - if "<" not in str(f.get("name", "")) or folder(f) not in covered + if "<" not in str(f.get("name", "")) or not is_answered(f, covered) ] + concrete @@ -913,6 +943,32 @@ def _companion_of(profile: dict, name: str) -> tuple[str, dict | None]: return "", None +def _image_beside( + companion_path: str, stem: str, image: dict | None, files_db: dict +) -> bool: + """Whether the folder of a companion holds an image it is read beside. + + PCSX2 opens `.rom1`, then the BIOS path with its extension + replaced (BiosTools.cpp LoadExtraRom, CDVD.cpp cdvdLoadNVRAM): the + companion's stem is the image's name or the image's own stem. The + image has to pass the image entry's test, its size and its ROM + directory, as the scan that selected it does. + """ + if image is None: + return False + directory = companion_path.rsplit("/", 1)[0] + "/" + wanted = stem.lower() + for _sha1, entry in _files_by_directory(files_db).get(directory, ()): + name = str(entry.get("name", "")) + if name.lower() != wanted and name.rpartition(".")[0].lower() != wanted: + continue + if name.lower() == companion_path.rsplit("/", 1)[-1].lower(): + continue + if shape_fits(image, entry.get("path", ""), entry.get("size", 0)): + return True + return False + + def _companion_extensions(profile: dict, seed_name: str) -> set[str]: """Extensions the profile declares beside a free-named BIOS entry. diff --git a/scripts/romdir.py b/scripts/romdir.py new file mode 100644 index 000000000..6f0ed8ad9 --- /dev/null +++ b/scripts/romdir.py @@ -0,0 +1,162 @@ +"""The PS2 ROM directory test the PCSX2 family runs on every BIOS candidate. + +A BIOS image starts with a table of 16-byte entries (a 10-byte name, a +16-bit extended info size, a 32-bit file size) whose first entry is RESET. +``LoadBiosVersion`` (pcsx2/ps2/BiosTools.cpp) reads the file 16 bytes at a +time from its start until an entry named RESET, then walks the table from +there, adding up the entry sizes rounded to 16 bytes to find where each +file of the ROM lies, and reads the 14 bytes of ROMVER at its offset. The +image is a BIOS when the walk met ROMVER. + +The forks differ in how they compare the names and in what an unreadable +read does, so a profile declares how its code reads the table (``romdir:`` +on the entry) and ``romdir_accepts`` runs that same walk. Stdlib only: +install.py carries a copy of the walk. +""" + +from __future__ import annotations + +import struct +from dataclasses import dataclass + +ENTRY_SIZE = 16 +NAME_SIZE = 10 +# for (i = 0; i < 512 * 1024; i++) in every LoadBiosVersion. +SEARCH_ENTRIES = 512 * 1024 +ROMVER_SIZE = 14 +EXTINFO_SIZE = 15 + +# What a read the code cannot complete does, by fork. +REJECT = "rejects" # wxWidgets streams throw (pxStreams.cpp, EndOfStream) +ENDS_WALK = "ends_walk" # the walk breaks, the result stands +IGNORED = "ignored" # AetherSX2 zero-fills romver and counts it found + +SPEC_KEYS = frozenset({ + "reset_compare", "missing_reset", "terminated_names", "extinfo", "unreadable_romver", + "truncated_walk", "offset_width", "refused_serials", "zones", "refused_zones", +}) + + +@dataclass(frozen=True) +class Romdir: + """What a walk that met ROMVER read: its 14 bytes and the EXTINFO serial.""" + + romver: bytes + serial: str + + +def _name_is(name: bytes, wanted: bytes, compared: int) -> bool: + """strncmp(name, wanted, compared) == 0 over a 10-byte field.""" + a = name[:compared] + b = (wanted + b"\x00")[:compared] + for x, y in zip(a, b.ljust(len(a), b"\x00")): + if x != y: + return False + if x == 0: + return True + return True + + +def walk(data: bytes, spec: dict) -> Romdir | None: + """The ROM directory walk one fork's LoadBiosVersion runs, or None. + + ``reset_compare`` is the length strncmp compares RESET over: 5 in the + wxWidgets and AetherSX2 code, which takes any name starting with RESET, + sizeof(fileName) (10) after, which wants RESET and a NUL. + ``terminated_names`` stops the walk at a name with no NUL in its 10 + bytes (strnlen check of the later code). ``missing_reset`` says what + 512 Ki entries read without RESET do: ``rejects`` where the code throws + (wxWidgets), ``ends_walk`` where the loop just ends and the walk starts + at the last entry read. ``extinfo`` reads 15 bytes at + the EXTINFO offset + 0x10 and ends the walk when they are not there. + ``unreadable_romver`` and ``truncated_walk`` say what a ROMVER read and + a next-entry read past the end of the file do: ``rejects``, + ``ends_walk`` or, for ROMVER, ``ignored``. ``offset_width`` is the + width of the offset sum: 32 where it is a uint (wxWidgets code), 64 + where it is an s64. + """ + size = len(data) + reset_compare = int(spec.get("reset_compare", NAME_SIZE)) + terminated = bool(spec.get("terminated_names", True)) + extinfo = bool(spec.get("extinfo", True)) + unreadable = spec.get("unreadable_romver", ENDS_WALK) + truncated = spec.get("truncated_walk", ENDS_WALK) + missing_reset = spec.get("missing_reset", ENDS_WALK) + offset_mask = (1 << int(spec.get("offset_width", 64))) - 1 + + pos = 0 + for _ in range(SEARCH_ENTRIES): + if pos + ENTRY_SIZE > size: + return None + name = data[pos:pos + NAME_SIZE] + pos += ENTRY_SIZE + if _name_is(name, b"RESET", reset_compare): + break + else: + if missing_reset == REJECT: + return None + + entry = data[pos - ENTRY_SIZE:pos] + offset = 0 + romver: bytes | None = None + serial = "" + while True: + name = entry[:NAME_SIZE] + if name[0] == 0 or (terminated and b"\x00" not in name): + break + if extinfo and _name_is(name, b"EXTINFO", NAME_SIZE): + start = offset + 0x10 + if start + EXTINFO_SIZE > size: + break + serial = data[start:start + EXTINFO_SIZE].split(b"\x00", 1)[0].decode("latin-1") + if _name_is(name, b"ROMVER", NAME_SIZE): + if offset + ROMVER_SIZE > size: + if unreadable == REJECT: + return None + if unreadable == ENDS_WALK: + break + romver = bytes(ROMVER_SIZE) + else: + romver = data[offset:offset + ROMVER_SIZE] + (file_size,) = struct.unpack_from(" size: + if truncated == REJECT: + return None + break + entry = data[pos:pos + ENTRY_SIZE] + pos += ENTRY_SIZE + if romver is None: + return None + return Romdir(romver, serial) + + +def zone_code(romver: bytes) -> str: + """The zone characters LoadBiosVersion switches on: ROMVER[4], and [5] + where the code reads it to tell COH-H (T then Z) from T10K.""" + return romver[4:6].decode("latin-1") + + +def romdir_accepts(data: bytes, spec: dict) -> bool: + """Whether a candidate passes the profile's ROM directory test. + + Past the walk, ``refused_serials`` turns away an EXTINFO serial + (ARMSX2's NOT_PS2_BIOS_STAMPS), and ``zones``/``refused_zones`` keep or + drop an image by its ROMVER zone, as a scan that wants a COH-H board + or passes one over does. A zone is written as the code tests it: ``J`` + is ROMVER[4], ``TZ`` ROMVER[4] and [5]. + """ + found = walk(data, spec) + if found is None: + return False + if found.serial in set(spec.get("refused_serials") or ()): + return False + code = zone_code(found.romver) + + def matches(zones: list | None) -> bool: + return any(code.startswith(str(z)) for z in zones or ()) + + if spec.get("zones") and not matches(spec["zones"]): + return False + return not matches(spec.get("refused_zones")) diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 3430d6cb3..c28cd2d82 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -95,7 +95,9 @@ _CHECK_FIELDS = { "md5": ("md5",), "sha1": ("sha1",), "sha256": ("sha256",), - "size": ("size", "min_size", "max_size"), + "size": ("size", "min_size", "max_size", "size_multiple"), + "romdir": ("romdir",), + "masked_sha1": ("masked_sha1",), } diff --git a/scripts/validation.py b/scripts/validation.py index 4e6daa4d8..e0e3aa668 100644 --- a/scripts/validation.py +++ b/scripts/validation.py @@ -15,10 +15,14 @@ from collections.abc import Callable, Iterator, Mapping, Sequence from common import ( adler32_of, archive_members, + CONTENT_CHECKS, compute_hashes, + content_fits, + declared_content_checks, entry_system_ids, name_match_size_ok, sanitize_pack_path, + shape_fits, size_fits, ) from hashing import md5_composite, parse_hash_list, parse_md5_list @@ -35,6 +39,11 @@ def _adler32_byteswapped(path: str) -> str: return adler32_of(path, byteswap=True) +_CONTENT_REASONS = { + "romdir": "romdir mismatch: no RESET and ROMVER the code accepts", + "masked_sha1": "masked_sha1 mismatch: the cleared image is not the one the code names", +} + # All reproducible validation types. _HASH_CHECKS = frozenset({"crc32", "md5", "sha1", "adler32"}) @@ -75,6 +84,7 @@ def _entry_rule(emu_name: str, f: dict, checks: set[str]) -> dict: - {None}, "min_size": f.get("min_size") if "size" in checks else None, "max_size": f.get("max_size") if "size" in checks else None, + "size_multiple": f.get("size_multiple") if "size" in checks else None, "crc32": values("crc32"), "md5": values("md5"), "sha1": values("sha1"), @@ -84,6 +94,9 @@ def _entry_rule(emu_name: str, f: dict, checks: set[str]) -> dict: }, "adler32_byteswap": bool(f.get("adler32_byteswap")), "crypto": {c for c in checks if c in _CRYPTO_CHECKS}, + "content": { + check: f.get(check) or {} for check in sorted(CONTENT_CHECKS & set(checks)) + }, # The checks whose failure the code only warns about: the file runs. "warn": warned_checks(f, checks), } @@ -122,6 +135,8 @@ def _rule_failure( return f"size too small: min {rule['min_size']}, got {actual_size}" if rule["max_size"] is not None and actual_size > rule["max_size"]: return f"size too large: max {rule['max_size']}, got {actual_size}" + if rule.get("size_multiple") and actual_size % int(rule["size_multiple"]): + return f"size mismatch: got {actual_size}, accepted multiples of {rule['size_multiple']}" for hash_type in ("crc32", "md5", "sha1", "sha256"): if rule[hash_type] and hash_type not in skip: if not hashes: @@ -139,6 +154,9 @@ def _rule_failure( if actual not in rule["adler32"]: accepted = ",".join(sorted(rule["adler32"])) return f"adler32 mismatch: got 0x{actual}, accepted [{accepted}]" + for check, spec in rule.get("content", {}).items(): + if check not in skip and not content_fits({"validation": [check], check: spec}, local_path): + return _CONTENT_REASONS[check] if rule["crypto"] - skip: from crypto_verify import check_crypto_validation @@ -253,6 +271,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: "sizes": set(), "min_size": None, "max_size": None, + "size_multiples": set(), "crc32": set(), "md5": set(), "sha1": set(), @@ -288,6 +307,8 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: index[fname]["min_size"] = ( min(cur, f["min_size"]) if cur is not None else f["min_size"] ) + if f.get("size_multiple"): + index[fname]["size_multiples"].add(int(f["size_multiple"])) if f.get("max_size") is not None: cur = index[fname]["max_size"] index[fname]["max_size"] = ( @@ -325,7 +346,7 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: # Per-emulator ground truth detail expected: dict = {} if "size" in checks: - for key in ("size", "min_size", "max_size"): + for key in ("size", "min_size", "max_size", "size_multiple"): if f.get(key) is not None: expected[key] = f[key] for hash_type in ("crc32", "md5", "sha1", "sha256"): @@ -334,6 +355,8 @@ def _build_validation_index(profiles: dict) -> dict[str, dict]: adler_val_pe = f.get("known_hash_adler32") or f.get("adler32") if adler_val_pe: expected["adler32"] = adler_val_pe + for check in sorted(CONTENT_CHECKS & set(checks)): + expected[check] = f.get(check) or {} pe_entry = { "checks": sorted(checks), "source_ref": f.get("source_ref"), @@ -479,6 +502,20 @@ def check_file_validation( if entry["max_size"] is not None and actual_size > entry["max_size"]: emus = _emulators_for_check("size", pe) return f"size too large: max {entry['max_size']}, got {actual_size}", emus + multiples = entry.get("size_multiples") or set() + if multiples and all(actual_size % m for m in multiples): + emus = _emulators_for_check("size", pe) + accepted = ",".join(str(m) for m in sorted(multiples)) + return f"size mismatch: got {actual_size}, accepted multiples of [{accepted}]", emus + + # A content test passes when one of the entries declaring it accepts + # the file, each read as its own code reads it. + for check in sorted(CONTENT_CHECKS & set(checks)): + specs = [r["content"][check] for r in entry.get("rules", []) if check in r.get("content", {})] + if specs and not any( + content_fits({"validation": [check], check: spec}, local_path) for spec in specs + ): + return _CONTENT_REASONS[check], _emulators_for_check(check, pe) # Hash checks: compute once, reuse for all hash types. # Each hash field is a set of accepted values (multiple valid ROM versions). @@ -855,8 +892,8 @@ def agnostic_substitute( held = candidate.get("path", "") if ( held.startswith(prefix) - and size_fits(entry, candidate.get("size", 0)) and os.path.exists(held) + and shape_fits(entry, held, candidate.get("size", 0)) ): return held, prefix break @@ -1148,8 +1185,12 @@ def core_size_refusal( if not local_path or not os.path.isfile(local_path): return None size = os.path.getsize(local_path) - if any(name_match_size_ok(entry, size) for entry in (file_entry, *alternatives)): + sized = [entry for entry in (file_entry, *alternatives) if name_match_size_ok(entry, size)] + if any(content_fits(entry, local_path) for entry in sized): return None + if sized: + tests = sorted({check for entry in sized for check in declared_content_checks(entry)}) + return f"the core's {', '.join(tests)} test turns it away" bounds = ( f"{file_entry['size']}" if file_entry.get("size") is not None else f"{file_entry.get('min_size', 0)} to {file_entry.get('max_size', 'any')}" diff --git a/scripts/verify.py b/scripts/verify.py index 9aaf6b4ca..081bb9d96 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -400,7 +400,7 @@ def _build_expected(file_entry: dict, checks: list[str]) -> dict: if not checks: return expected if "size" in checks: - for key in ("size", "min_size", "max_size"): + for key in ("size", "min_size", "max_size", "size_multiple"): if file_entry.get(key) is not None: expected[key] = file_entry[key] for hash_type in ("crc32", "md5", "sha1", "sha256"): diff --git a/tests/test_bios_companions.py b/tests/test_bios_companions.py index d0b8ba4c0..55b3f94e9 100644 --- a/tests/test_bios_companions.py +++ b/tests/test_bios_companions.py @@ -79,7 +79,9 @@ class Companions(unittest.TestCase): } stored = { "ps2-0230a-20080220.bin": b"I" * 8, + "SCPH-70004.BIN": b"B" * 8, "SCPH-70004.ROM1": b"R" * 6, + "246C.ROM1": b"N" * 3, "cheats_ws.zip": b"Z", "GameIndex.yaml": b"Y", } @@ -107,6 +109,8 @@ class Companions(unittest.TestCase): names = {e["destination"] for e in extras} self.assertIn("SCPH-70004.rom1", names) self.assertFalse({"SCPH-70004.ROM1", "cheats_ws.zip", "GameIndex.yaml"} & names) + # Read beside the image the core selected only: no 246C image here. + self.assertNotIn("246C.rom1", names) def test_a_shared_image_brings_each_cores_companions(self): """Recalbox runs LRPS2 and the old libretro PCSX2 on one ps2/ folder. diff --git a/tests/test_content_checks.py b/tests/test_content_checks.py new file mode 100644 index 000000000..a72cb90cf --- /dev/null +++ b/tests/test_content_checks.py @@ -0,0 +1,322 @@ +"""Content tests a profile declares, run the way the emulator runs them. + +The PCSX2 family lists every file of its BIOS folder whose ROM directory +holds RESET and ROMVER, whatever its name and, in most forks, its size. A +4 to 8 MB bound stood in for that test on Recalbox's libretro PCSX2: it +dropped the 2 MB System 246 image the core lists and let the scan carry +.nvm and .mec files that sit beside no image. Amiberry names a flash image +by the SHA-1 of its bytes with the board configuration cleared, xemu maps +any flash image whose size is a whole number of 64 KiB. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import hashlib +import struct +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import common # noqa: E402 +import generate_pack as builder # noqa: E402 +import packextras # noqa: E402 +from romdir import romdir_accepts, walk # noqa: E402 +from validation import _build_validation_index, check_file_validation # noqa: E402 + +# How each fork reads the table, as the profiles declare it. +WX = {"reset_compare": 5, "missing_reset": "rejects", "terminated_names": False, + "extinfo": False, "unreadable_romver": "rejects", "truncated_walk": "rejects", + "offset_width": 32} +AETHER = {"reset_compare": 5, "missing_reset": "ends_walk", "terminated_names": False, + "extinfo": False, "unreadable_romver": "ignored", "truncated_walk": "ends_walk"} +CLASSIC = {"reset_compare": 10, "missing_reset": "ends_walk", "terminated_names": True, + "extinfo": False, "unreadable_romver": "ends_walk", "truncated_walk": "ends_walk"} +MODERN = {"reset_compare": 10, "missing_reset": "ends_walk", "terminated_names": True, + "extinfo": True, "unreadable_romver": "ends_walk", "truncated_walk": "ends_walk"} +FORKS = {"wx": WX, "aether": AETHER, "classic": CLASSIC, "modern": MODERN} + +ROMDIR_AT = 0x2700 # where retail images keep the table, inside RESET + + +def entry(name: bytes, size: int) -> bytes: + return name.ljust(10, b"\x00")[:10] + struct.pack(" bytes: + """A BIOS laid out as a retail one: the table at 0x2700 inside RESET, + then ROMDIR, EXTINFO and ROMVER, each file at the sum of the sizes + before it rounded to 16 bytes.""" + if entries is None: + entries = [(b"RESET", ROMDIR_AT), (b"ROMDIR", 0x50), (b"EXTINFO", 0x30), (b"ROMVER", 0x10)] + table = b"".join(entry(n, s) for n, s in entries) + (entry(b"", 0) if terminator else b"") + offsets, offset = {}, 0 + for name, size in entries: + offsets[name] = offset + offset += size if size % 16 == 0 else (size + 16) & 0xFFFFFFF0 + end = max(ROMDIR_AT + len(table), min(offset, 1 << 20)) + tail + data = bytearray(end) + data[ROMDIR_AT:ROMDIR_AT + len(table)] = table + for name, payload, skip in ((b"EXTINFO", serial, 0x10), (b"ROMVER", romver, 0)): + at = offsets.get(name, 1 << 40) + skip + if at + len(payload) <= len(data): + data[at:at + len(payload)] = payload + return bytes(data) + + +class RomdirWalk(unittest.TestCase): + def test_a_retail_layout_passes_every_fork(self): + data = image() + for fork, spec in FORKS.items(): + with self.subTest(fork=fork): + found = walk(data, spec) + self.assertIsNotNone(found) + self.assertEqual(found.romver, b"0230AC20080220") + self.assertEqual(walk(data, MODERN).serial, "20080220-000000") + + def test_a_file_without_romver_fails_every_fork(self): + data = image([(b"RESET", ROMDIR_AT), (b"ROMDIR", 0x40), (b"EXTINFO", 0x30)]) + for fork, spec in FORKS.items(): + with self.subTest(fork=fork): + self.assertFalse(romdir_accepts(data, spec)) + self.assertFalse(romdir_accepts(b"\x00" * 0x4000, MODERN)) + self.assertFalse(romdir_accepts(b"", MODERN)) + + def test_reset_compare_is_the_strncmp_length(self): + """strncmp(name, "RESET", 5) in the wxWidgets and AetherSX2 code takes + RESETX, sizeof(fileName) wants RESET and a NUL.""" + data = image([(b"RESETX", ROMDIR_AT), (b"ROMDIR", 0x40), (b"ROMVER", 0x10)]) + self.assertTrue(romdir_accepts(data, WX)) + self.assertTrue(romdir_accepts(data, AETHER)) + self.assertFalse(romdir_accepts(data, MODERN)) + + def test_a_name_without_nul_ends_the_later_walk(self): + data = image([(b"RESET", ROMDIR_AT), (b"ABCDEFGHIJ", 0x100), (b"ROMVER", 0x10)]) + self.assertTrue(romdir_accepts(data, WX)) + self.assertTrue(romdir_accepts(data, AETHER)) + self.assertFalse(romdir_accepts(data, CLASSIC)) + self.assertFalse(romdir_accepts(data, MODERN)) + + def test_romver_past_the_end_of_the_file(self): + """AetherSX2 zero-fills the 14 bytes and counts ROMVER found, the + wxWidgets stream throws, the later code ends the walk.""" + data = image([(b"RESET", ROMDIR_AT), (b"ROMDIR", 0x200000), (b"ROMVER", 0x10)]) + self.assertTrue(romdir_accepts(data, AETHER)) + self.assertFalse(romdir_accepts(data, WX)) + self.assertFalse(romdir_accepts(data, CLASSIC)) + + def test_an_unreadable_extinfo_ends_the_walk_before_romver(self): + data = image([(b"RESET", ROMDIR_AT), (b"ROMDIR", 0x200000), (b"EXTINFO", 0x30), + (b"ROMVER", 0x10)]) + self.assertFalse(romdir_accepts(data, MODERN)) + self.assertTrue(romdir_accepts(data, AETHER)) + + def test_a_table_that_runs_off_the_file(self): + """Without the zero entry the next read fails: the wxWidgets stream + throws, the later code keeps the ROMVER it met.""" + data = image([(b"RESET", 0x10), (b"ROMVER", 0x10)], terminator=False, tail=0) + self.assertEqual(len(data), ROMDIR_AT + 2 * 16) + self.assertTrue(romdir_accepts(data, MODERN)) + self.assertFalse(romdir_accepts(data, WX)) + + def test_the_wx_offset_is_a_32_bit_uint(self): + wrap = (1 << 32) - ROMDIR_AT - 0x50 + data = image([(b"RESET", ROMDIR_AT), (b"ROMDIR", 0x50), (b"PAD", wrap), (b"ROMVER", 0x10)]) + romver_at = 0 + data = bytearray(data) + data[romver_at:romver_at + 14] = b"0230AC20080220" + self.assertTrue(romdir_accepts(bytes(data), WX)) + self.assertFalse(romdir_accepts(bytes(data), {**WX, "offset_width": 64})) + + def test_512_ki_entries_without_reset(self): + """The later loop just ends and walks on from the last entry it read, + the wxWidgets code throws.""" + data = bytearray(512 * 1024 * 16) + data[-16:] = entry(b"ROMVER", 0x10) + data[0:14] = b"0230AC20080220" + self.assertTrue(romdir_accepts(bytes(data), MODERN)) + self.assertFalse(romdir_accepts(bytes(data), WX)) + + def test_serial_and_zone_tests(self): + coh = image(romver=b"0100TZ20000306", serial=b"20040519-145634") + stamp = image(serial=b"20111012-094243") + self.assertTrue(romdir_accepts(coh, {**MODERN, "zones": ["TZ"]})) + self.assertFalse(romdir_accepts(image(), {**MODERN, "zones": ["TZ"]})) + self.assertFalse(romdir_accepts(coh, {**MODERN, "refused_zones": ["TZ"]})) + self.assertTrue(romdir_accepts(image(romver=b"0100TC20000306"), + {**MODERN, "refused_zones": ["TZ"]})) + self.assertFalse(romdir_accepts(stamp, {**MODERN, "refused_serials": ["20111012-094243"]})) + + def test_every_pcsx2_profile_declares_its_walk(self): + """A content-found PS2 image without its walk would be judged by the + defaults, not by what its fork does.""" + import yaml + for path in sorted((REPO_ROOT / "emulators").glob("*.yml")): + profile = yaml.safe_load(path.read_text()) + if "sony-playstation-2" not in (profile.get("systems") or []): + continue + for f in profile.get("files") or []: + if "romdir" in (f.get("validation") or []): + with self.subTest(profile=path.stem, name=f.get("name")): + self.assertTrue(set(f["romdir"]) >= { + "reset_compare", "missing_reset", "terminated_names", "extinfo", + "unreadable_romver", "truncated_walk"}) + + +class OtherContentTests(unittest.TestCase): + def test_masked_sha1_clears_the_config_bytes(self): + payload = bytearray(range(256)) * 512 # 131072 bytes + cleared = bytearray(payload) + cleared[0x10:0x30] = bytes(0x20) + spec = {"zeroed": [16, 48], "reads": [{"size": 131072, "lengths": [0x1C000]}], + "sha1": hashlib.sha1(cleared[:0x1C000]).hexdigest()} + entry_ = {"validation": ["masked_sha1"], "masked_sha1": spec} + with tempfile.TemporaryDirectory() as tmp: + good = Path(tmp, "flash.rom") + good.write_bytes(bytes(payload)) + other = Path(tmp, "other.rom") + other.write_bytes(bytes(payload[:-1]) + b"\x01") + short = Path(tmp, "short.rom") + short.write_bytes(bytes(payload[:65536])) + config = bytearray(payload) + config[0x20] ^= 0xFF + reconfigured = Path(tmp, "reconfigured.rom") + reconfigured.write_bytes(bytes(config)) + self.assertTrue(common.content_fits(entry_, str(good))) + self.assertTrue(common.content_fits(entry_, str(reconfigured))) + self.assertTrue(common.content_fits(entry_, str(other))) # past 0x1c000 + self.assertFalse(common.content_fits(entry_, str(short))) + payload[0x100] ^= 1 + good.write_bytes(bytes(payload)) + common._masked_sha1_verdict.cache_clear() + self.assertFalse(common.content_fits(entry_, str(good))) + + def test_size_multiple(self): + flash = {"min_size": 65536, "size_multiple": 65536, "validation": ["size"]} + self.assertTrue(common.size_fits(flash, 262144)) + self.assertTrue(common.size_fits(flash, 65536 * 3)) + self.assertFalse(common.size_fits(flash, 65536 + 1)) + self.assertFalse(common.size_fits(flash, 0)) + + +class ScannerVerifyManifestAgree(unittest.TestCase): + """The pack and manifest scan, the emulator pack and verify --emulator + expansion, the name step of the resolver and verify's validation judge + one folder the same way.""" + + IMAGES = { + "SCPH-70004.BIN": image(), + "246C.bin": image(romver=b"0100TZ20000306", serial=b"20021119-163841"), + "GameIndex.yaml": b"GameIndex: not a BIOS\n" * 300, + "ps2-dvd-311.bin": b"\x00" * 0x4000, + } + COMPANIONS = {"SCPH-70004.nvm": b"N" * 1024, "246C.mec": b"\x03\x06\x02\x00", + "loose.nvm": b"L" * 1024} + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + (self.root / "emulators").mkdir() + files, by_name = {}, {} + for name, payload in {**self.IMAGES, **self.COMPANIONS}.items(): + path = self.root / "bios" / "PS2" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + sha1 = hashlib.sha1(payload).hexdigest() + files[sha1] = {"path": str(path), "name": name, "size": len(payload), "sha1": sha1, + "md5": hashlib.md5(payload).hexdigest()} + by_name.setdefault(name, []).append(sha1) + self.sha1 = {r["name"]: s for s, r in files.items()} + self.db = {"files": files, "indexes": { + "by_name": by_name, "by_md5": {r["md5"]: s for s, r in files.items()}, + "by_sha256": {}, "by_crc32": {}, "by_path_suffix": {}}} + spec = "\n".join(f" {k}: {str(v).lower() if isinstance(v, bool) else v}" + for k, v in MODERN.items()) + # The anchor proves bios/PS2 holds this system's BIOS by content. + (self.root / "emulators" / "anchor.yml").write_text( + "emulator: anchor\ntype: libretro\nsystems: [sony-playstation-2]\ncores: [anchor]\n" + "files:\n - name: SCPH-70004.BIN\n" + f" sha1: {self.sha1['SCPH-70004.BIN']}\n" + ) + (self.root / "emulators" / "demo.yml").write_text( + "emulator: demo\ntype: libretro\nbios_mode: agnostic\n" + "systems: [sony-playstation-2]\ncores: [demo]\nfiles:\n" + " - name: \".bin\"\n found_by: content\n path: \"ps2/.bin\"\n" + " required: true\n validation: [romdir]\n romdir:\n" + spec + "\n" + " - name: \".nvm\"\n path: \"ps2/.nvm\"\n size: 1024\n" + " - name: \".mec\"\n path: \"ps2/.mec\"\n size: 4\n" + ) + common._emulator_profiles_cache.clear() + self.profiles = common.load_emulator_profiles(str(self.root / "emulators")) + self.config = {"platform": "Demo", "verification_mode": "existence", + "cores": ["demo", "anchor"], "base_destination": "", "systems": {}} + + def tearDown(self): + common._emulator_profiles_cache.clear() + self._tmp.cleanup() + + def accepted(self) -> set[str]: + """What the core lists: each image its walk accepts, each companion + beside an accepted image of its stem.""" + seed = self.profiles["demo"]["files"][0] + images = {n for n in self.IMAGES + if common.content_fits(seed, str(self.root / "bios" / "PS2" / n))} + stems = {n.rsplit(".", 1)[0] for n in images} + return images | {n for n in self.COMPANIONS if n.rsplit(".", 1)[0] in stems} + + def test_the_walk_settles_the_folder(self): + self.assertEqual(self.accepted(), {"SCPH-70004.BIN", "246C.bin", + "SCPH-70004.nvm", "246C.mec"}) + + def test_platform_pack_and_manifest_scan(self): + extras = builder._collect_emulator_extras( + self.config, str(self.root / "emulators"), self.db, set(), "", self.profiles, + ) + placed = {e["name"] for e in extras if e.get("source_profile") == "demo"} + self.assertEqual(placed, self.accepted()) + self.assertTrue(all(e["destination"].startswith("ps2/") for e in extras + if e.get("source_profile") == "demo")) + + def test_emulator_pack_and_verify_expansion(self): + demo = self.profiles["demo"] + expanded = packextras.expand_agnostic_entries( + demo["files"], "demo", demo, self.db, False, self.profiles, + ) + names = {f["name"] for f in expanded if f.get("agnostic_scan")} + self.assertEqual(names, self.accepted()) + + def test_resolver_name_step_and_validation_agree(self): + index = _build_validation_index(self.profiles_with_labels()) + for name in self.IMAGES: + path = str(self.root / "bios" / "PS2" / name) + entry_ = {"name": name, "found_by": "content", "validation": ["romdir"], + "romdir": MODERN, "source_profile": "labels"} + resolved, status = common.resolve_local_file(entry_, self.db) + failure = check_file_validation(path, name, index) + with self.subTest(name=name): + accepted = name in self.accepted() + # A name the walk refuses is a mismatch, never a name match. + self.assertEqual(resolved, path) + self.assertEqual(status, "name_exact" if accepted else "hash_mismatch") + self.assertEqual(failure is None, accepted) + + def profiles_with_labels(self) -> dict: + return {"labels": {"emulator": "labels", "files": [ + {"name": name, "found_by": "content", "validation": ["romdir"], "romdir": MODERN} + for name in self.IMAGES + ]}} + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/profiling.md b/wiki/profiling.md index 98d902053..ccda747d3 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -615,10 +615,13 @@ look. | `archive` | ROM set the file lives inside, for arcade entries (e.g. `neogeo.zip`) | | `size` | expected size in bytes, or a list when the code accepts several exact sizes | | `min_size`, `max_size` | size range when the code accepts a range | +| `size_multiple` | granule the size must be a whole number of, the bounds still applying (xemu maps any flash image of a positive multiple of 64 KiB) | | `md5`, `sha1`, `crc32`, `sha256` | expected hashes from source code | | `known_hash_adler32`, `adler32_byteswap` | expected Adler-32 hash (Dolphin's DSP ROMs). Pair with `adler32_byteswap` when the code hashes 16-bit byte-swapped data | | `adler32_pairs_with`, `adler32_pairs` | the file hashed together with this one and the combinations the code accepts, this file's value first, in table order: Dolphin warns on a DSP ROM pair outside its `known_roms` even when each file passes alone | -| `validation` | checks the code performs: `size`, `crc32`, `md5`, `sha1`, `adler32`, `signature`, `crypto`. Can be a list or dict `{core: [...], upstream: [...]}` for divergent checks | +| `validation` | checks the code performs: `size`, `crc32`, `md5`, `sha1`, `adler32`, `romdir`, `masked_sha1`, `signature`, `crypto`. Can be a list or dict `{core: [...], upstream: [...]}` for divergent checks | +| `romdir` | how the code reads a PS2 ROM directory for `validation: [romdir]`: `reset_compare` (5 or 10 bytes compared with RESET), `missing_reset`, `terminated_names`, `extinfo`, `unreadable_romver`, `truncated_walk`, `offset_width`, and `refused_serials`, `zones`, `refused_zones` for the tests after the walk. Each is read in the fork's `LoadBiosVersion` | +| `masked_sha1` | for `validation: [masked_sha1]`: the bytes the code clears (`zeroed`), the lengths it hashes for each file size it tries (`reads`) and the SHA-1 it compares (Amiberry's CyberStorm and Blizzard PPC flash images) | | `aliases` | alternate filenames for the same file | | `mode` | `libretro`, `standalone`, or `both` | | `hle_fallback` | true if a high-level emulation path exists | diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index c9f7faa5d..6ebf809b3 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -258,6 +258,8 @@ Each entry in the index tracks: - `min_size`, `max_size`: bounds when the code accepts a range - `crc32`, `md5`, `sha1`, `sha256`: sets of accepted hash values - `adler32`: set of accepted Adler-32 values +- `size_multiples`: granules a size must be a whole number of +- `rules[].content`: the content tests of each entry (`romdir`, `masked_sha1`) with their spec - `crypto_only`: non-reproducible checks (see below) - `per_emulator`: per-core detail with source references @@ -274,6 +276,13 @@ console-specific cryptographic keys (e.g., RSA-2048 for 3DS, AES-128-CBC for cer firmware). `verify.py` reports these as informational but cannot verify them without the keys. Size checks still apply if combined with crypto. +**Content tests** (`CONTENT_CHECKS` in `common.py`): `romdir` walks a PS2 ROM +directory the way the profile's fork reads it (`scripts/romdir.py`), and +`masked_sha1` hashes a file with a byte range cleared. `content_fits()` runs them, +and the name steps of the resolver, the folder scans of the pack and manifest, +the emulator expansion and `verify.py` all call it, so they accept and refuse the +same files. + ### Size validation Three forms: