From 33ae27ebcfb5d72278ff1c241ce671a1ab70d117 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:20:14 +0200 Subject: [PATCH] fix: count an emulator pack by slot, as verify --- scripts/generate_pack.py | 151 +++++++++++++++++++---------- scripts/verify.py | 21 ++-- tests/test_emulator_pack_counts.py | 57 +++++++++++ 3 files changed, 170 insertions(+), 59 deletions(-) create mode 100644 tests/test_emulator_pack_counts.py diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 7298bafc..b2991890 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -1363,6 +1363,8 @@ def generate_emulator_pack( missing_files = [] rejected_files: list[str] = [] discrepancies: list[str] = [] + unsourceable_files: list[str] = [] + instruction_files: list[str] = [] seen_destinations: set[str] = set() seen_lower: set[str] = set() seen_parents: set[str] = ( @@ -1461,49 +1463,107 @@ def generate_emulator_pack( else: missing_files.append(archive_name) - # Pack individual files (skip archived ones) + # Pack individual files (skip archived ones). The entries of one + # profile on one destination are the releases the code accepts + # there (raze's seven SW.GRP): the first one the core accepts + # fills the slot, as verify --emulator credits it, and the slot + # has one outcome however many alternatives it lists. + slots: dict[str, list[dict]] = {} for fe in files: if required_only and fe.get("required") is False: continue if fe.get("archive"): continue - dest = _resolve_destination(fe, pack_structure, standalone) if not dest: continue if region_drops and dest in region_drops: continue + slots.setdefault(dest, []).append(fe) + for dest, alternatives in slots.items(): if dest.lower() in seen_lower: continue if _has_path_conflict(dest, seen_destinations, seen_parents): continue - storage = fe.get("storage", "embedded") - if storage == "user_provided": + provided = next( + (fe for fe in alternatives if fe.get("storage") == "user_provided"), + None, + ) + if provided is not None: + # Instructions, not a file: written beside the slot the + # user fills, and counted apart from what the pack holds. seen_destinations.add(dest) _register_path(dest, seen_destinations, seen_parents) seen_lower.add(dest.lower()) - instr = fe.get("instructions", "Please provide this file manually.") - instr_name = f"INSTRUCTIONS_{fe['name']}.txt" + instr = provided.get("instructions", "Please provide this file manually.") + folder = dest.rsplit("/", 1)[0] + "/" if "/" in dest else "" _write_generated_member( - zf, instr_name, f"File needed: {fe['name']}\n\n{instr}\n" + zf, + f"{folder}INSTRUCTIONS_{provided['name']}.txt", + f"File needed: {provided['name']}\n\n{instr}\n", + ) + # verify --emulator counts an unsourceable one as such. + (unsourceable_files if provided.get("unsourceable") else instruction_files).append( + provided["name"] ) - total_files += 1 continue - dest_hint = fe.get("path", "") - local_path, status = resolve_file( - {**fe, "source_profile": emu_name}, - db, - bios_dir, - zip_contents, - dest_hint=dest_hint, - data_dir_registry=data_registry, - offline=offline, - ) + chosen: tuple[dict, str, str | None] | None = None + fallback: tuple[dict, str, str | None] | None = None + refusal: str | None = None + for fe in alternatives: + local_path, status = resolve_file( + {**fe, "source_profile": emu_name}, + db, + bios_dir, + zip_contents, + dest_hint=fe.get("path", ""), + data_dir_registry=data_registry, + offline=offline, + ) + if status == "external": + chosen = (fe, "", "external") + break + if status in ("not_found", "user_provided") or not local_path: + continue + # The file verify --emulator credits is the one shipped: + # a dump the core's own check rejects gives way to a held + # one it accepts (azahar's otp.bin, dolphin's dsp_rom.bin). + refused = check_file_validation( + local_path, fe["name"], validation_index, bios_dir, {emu_name} + ) + if refused: + better = find_validated_variant( + fe, db, local_path, validation_index, bios_dir, + emulators={emu_name}, + ) + if not better: + refusal = refusal or f"{fe['name']} -{refused[0]}" + continue + chosen = (fe, better, None) + break + if status == "hash_mismatch": + # Not a check the core runs, so the file still serves; + # it is not the dump the profile documents. + fallback = fallback or (fe, local_path, "mismatch") + continue + chosen = (fe, local_path, None) + break + chosen = chosen or fallback - if status == "external": + if chosen is None: + if refusal: + rejected_files.append(refusal) + elif all(fe.get("unsourceable") for fe in alternatives): + unsourceable_files.append(alternatives[0]["name"]) + else: + missing_files.append(alternatives[0]["name"]) + continue + + fe, local_path, how = chosen + if how == "external": file_ext = os.path.splitext(fe["name"])[1] or "" with tempfile.NamedTemporaryFile( delete=False, suffix=file_ext @@ -1522,31 +1582,7 @@ def generate_emulator_pack( if os.path.exists(tmp_path): os.unlink(tmp_path) continue - - if status in ("not_found", "user_provided") or not local_path: - missing_files.append(fe["name"]) - continue - - # The file verify --emulator credits is the one shipped: a - # dump the core's own check rejects gives way to a held one - # it accepts (azahar's otp.bin, dolphin's dsp_rom.bin). With - # none, the core would refuse what the pack ships: it is named - # and left out, as verify reports it. - refused = check_file_validation( - local_path, fe["name"], validation_index, bios_dir - ) - if refused: - better = find_validated_variant( - fe, db, local_path, validation_index, bios_dir - ) - if not better: - reason, _owners = refused - rejected_files.append(f"{fe['name']} -{reason}") - continue - local_path = better - elif status == "hash_mismatch": - # Not a check the core runs, so the file still serves; - # it is not the dump the profile documents, and says so. + if how == "mismatch": discrepancies.append( f"{fe['name']} -not the dump the {emu_name} profile declares" ) @@ -1554,12 +1590,10 @@ def generate_emulator_pack( # SHA1 dedup: skip if same physical file AND same destination # (but allow same file to be packed under different destinations, # e.g., IPL.bin in GC/USA/ and GC/EUR/ from same source) - if local_path: - real = os.path.realpath(local_path) - dedup_key_hash = f"{real}:{dest}" - if dedup_key_hash in seen_hashes: - continue - seen_hashes.add(dedup_key_hash) + dedup_key_hash = f"{os.path.realpath(local_path)}:{dest}" + if dedup_key_hash in seen_hashes: + continue + seen_hashes.add(dedup_key_hash) if local_path.endswith(".zip"): _add_zip_to_pack(local_path, dest, zf, fe) @@ -1571,7 +1605,7 @@ def generate_emulator_pack( total_files += 1 # Remove empty ZIP (no files packed and no missing = nothing to ship) - if total_files == 0 and not missing_files and not rejected_files: + if total_files == 0 and not (missing_files or rejected_files or instruction_files or unsourceable_files): os.unlink(zip_path) # Report @@ -1583,9 +1617,17 @@ def generate_emulator_pack( parts.append(f"{missing_count} missing") if rejected_files: parts.append(f"{len(rejected_files)} refused by the core") + if unsourceable_files: + parts.append(f"{len(unsourceable_files)} unsourceable") + if instruction_files: + parts.append(f"{len(instruction_files)} to provide") print(f" {zip_path}: {', '.join(parts)}") for name in missing_files: print(f" MISSING: {name}") + for name in unsourceable_files: + print(f" UNSOURCEABLE: {name}") + for name in instruction_files: + print(f" TO PROVIDE: {name} (instructions in the pack)") for line in rejected_files: print(f" REFUSED: {line}") for line in discrepancies: @@ -1595,7 +1637,12 @@ def generate_emulator_pack( f" Note: data directory '{ref}' required but not included (use refresh_data_dirs.py)" ) - return zip_path if total_files > 0 or missing_files or rejected_files else None + return ( + zip_path + if total_files > 0 or missing_files or rejected_files or instruction_files + or unsourceable_files + else None + ) def generate_system_pack( diff --git a/scripts/verify.py b/scripts/verify.py index cf513ee6..ea4ece15 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1585,11 +1585,15 @@ def verify_emulator( ) else: # Apply emulator validation - check = check_file_validation(local_path, name, validation_index) + # Judged by this emulator's own entries, as it runs its check. + check = check_file_validation( + local_path, name, validation_index, emulators={emu_name} + ) if check: reason, _emus = check better = find_validated_variant( file_entry, db, local_path, validation_index, + emulators={emu_name}, ) if better: result = { @@ -1644,6 +1648,7 @@ def verify_emulator( if held and len(alternatives) > 1: alternatives = held[:1] for result, required, hle in alternatives: + result["destination"] = dest details.append(result) # Aggregate by destination (path if available, else name) @@ -1745,12 +1750,14 @@ def print_emulator_result(result: dict, verbose: bool = False) -> None: sc = result.get("status_counts", {}) untested = sc.get(Status.UNTESTED, 0) missing = sc.get(Status.MISSING, 0) - unsourceable_names = { - d["name"] - for d in result["details"] - if d["status"] == Status.MISSING and "unsourceable" in d - } - unsourceable = len(unsourceable_names) + # Counted by slot, as missing is: xash3d's thirteen unsourceable + # pak0.pak and liblist.gam were three names and ten "missing". + missing_slots: dict[str, bool] = {} + for d in result["details"]: + if d["status"] == Status.MISSING: + slot = d.get("destination") or d["name"] + missing_slots[slot] = missing_slots.get(slot, True) and "unsourceable" in d + unsourceable = sum(missing_slots.values()) parts = [f"{ok_count}/{total} OK"] if untested: parts.append(f"{untested} untested") diff --git a/tests/test_emulator_pack_counts.py b/tests/test_emulator_pack_counts.py new file mode 100644 index 00000000..e1383172 --- /dev/null +++ b/tests/test_emulator_pack_counts.py @@ -0,0 +1,57 @@ +"""An emulator pack and verify --emulator count the same slots the same way. + +Accepted alternatives on one destination (raze's SW.GRP), unsourceable +entries and user-provided ones were counted one way by the pack and another +by verify: raze was "10 packed, 29 missing, 4 refused" against "10/23 OK, +1 untested, 12 missing", xash3d's thirteen unsourceable slots were three +names and ten "missing". +""" + +from __future__ import annotations + +import re +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +CASES = ("raze", "rust_dos", "hatari", "xash3d_fwgs", "catacombgl", "openjk", "sonic2013") + + +def _count(text: str, word: str) -> int: + match = re.search(rf"(\d+) {word}", text) + return int(match.group(1)) if match else 0 + + +class PackAndReportAgree(unittest.TestCase): + def test_each_emulator_pack_counts_as_verify_does(self): + if not (REPO_ROOT / "database.json").is_file(): + self.skipTest("database.json is not built") + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + for emulator in CASES: + with self.subTest(emulator=emulator), tempfile.TemporaryDirectory( + dir=REPO_ROOT / "tmp" + ) as out: + pack = subprocess.run( + [sys.executable, "scripts/generate_pack.py", "--emulator", emulator, + "--offline", "--output-dir", out], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, check=True, + ).stdout + report = subprocess.run( + [sys.executable, "scripts/verify.py", "--emulator", emulator], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, check=False, + ).stdout + summary = next(line for line in pack.splitlines() if "_BIOS_Pack.zip:" in line) + verdict = next(line for line in report.splitlines() if line.startswith(f"{emulator}:")) + ok = int(re.search(r"(\d+)/\d+ OK", verdict).group(1)) + self.assertEqual(_count(summary, "files packed"), ok, (summary, verdict)) + self.assertEqual(_count(summary, "missing"), _count(verdict, "missing")) + self.assertEqual(_count(summary, "unsourceable"), _count(verdict, "unsourceable")) + self.assertEqual(_count(summary, "refused"), _count(verdict, "untested")) + + +if __name__ == "__main__": + unittest.main()