diff --git a/scripts/common.py b/scripts/common.py index 231e835d6..47d53833d 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -664,8 +664,15 @@ def load_target_config( targets_dir = os.path.join(platforms_dir, "targets") target_file = os.path.join(targets_dir, f"{platform_name}.yml") if not os.path.exists(target_file): + # The expected state for a platform that ships one build, said as + # such: the file path read like a file missing from the repository. + with_targets = sorted( + Path(f).stem for f in os.listdir(targets_dir) + if f.endswith(".yml") and not f.startswith("_") + ) raise FileNotFoundError( - f"No target config for platform '{platform_name}': {target_file}" + f"{platform_name} has no hardware targets: drop --target " + f"(platforms with targets: {', '.join(with_targets)})" ) with open(target_file) as f: data = yaml_load(f) or {} @@ -2986,6 +2993,46 @@ def filter_systems_by_target( } +def platform_scope_systems( + config: dict, + profiles: dict[str, dict], + target_cores: set[str] | None, + system_filter: list[str] | None = None, +) -> dict[str, dict]: + """The systems a pack or a report of a platform covers. + + Those a core of the target runs, then the ones a --system list keeps. + verify.py and generate_pack.py read this one function. + """ + plat_cores = resolve_platform_cores(config, profiles) if target_cores else None + systems = filter_systems_by_target( + config.get("systems", {}), profiles, target_cores, platform_cores=plat_cores + ) + return select_systems(systems, system_filter) if system_filter else systems + + +def system_scope_error( + config: dict, platform_name: str, system_filter: list[str], target: str | None +) -> str: + """Why a --system list leaves a platform no system. + + A system the platform declares but no core of the target runs was + reported as a system the platform lacks, with a lookalike to try and a + listing that refuses --target. + """ + systems = config.get("systems", {}) + declared = select_systems(systems, system_filter) + if declared and target: + return ( + f"no core {platform_name} runs on target {target} emulates " + f"{', '.join(sorted(declared))}" + ) + hint = suggest_names( + system_filter, sorted(systems), f"--platform {platform_name} --list-systems" + ) + return f"{platform_name} has no system matching {', '.join(system_filter)}. {hint}" + + def platform_declarations( config: dict, db: dict, *, enrich: bool = True ) -> dict[str, list[tuple[int | None, str]]]: @@ -3145,6 +3192,22 @@ def list_system_ids(emulators_dir: str) -> None: print(f" {spelling:35s} ({count} emulator{'s' if count > 1 else ''})") +def print_platform_listing(platforms_dir: str) -> None: + """Every registered platform, one per line, marked when archived or + served by the installer only. + + Every platform takes --platform. --all covers the active ones, and the + pack builder's --all the release packs. Printed by --list-platforms in + verify.py and generate_pack.py, the name install.py gives it. + """ + active = set(list_registered_platforms(platforms_dir)) + released = set(list_release_pack_platforms(platforms_dir, include_archived=True)) + for name in list_registered_platforms(platforms_dir, include_archived=True): + marks = [m for m, on in (("archived", name not in active), + ("installer only", name not in released)) if on] + print(f"{name} ({', '.join(marks)})" if marks else name) + + def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None: """Print system IDs from a platform's YAML config, with the platform's own names for them, which --system also accepts.""" diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 07dc00ecc..1bed3b5e3 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -53,7 +53,10 @@ from common import ( list_platform_system_ids, platform_system_names, resolve_system_arg, + print_platform_listing, select_systems, + platform_scope_systems, + system_scope_error, canonical_target_name, platform_files_target, list_registered_platforms, @@ -753,31 +756,13 @@ def generate_pack( target_cores=target_cores, source=source, ) - # Filter systems by target if specified - plat_cores = ( - resolve_platform_cores(config, emu_profiles or {}) if target_cores else None + pack_systems = platform_scope_systems( + config, emu_profiles or {}, target_cores, system_filter ) - pack_systems = filter_systems_by_target( - config.get("systems", {}), - emu_profiles or {}, - target_cores, - platform_cores=plat_cores, - ) - - if system_filter: - filtered = select_systems(pack_systems, system_filter) - if not filtered: - hint = suggest_names( - system_filter, - sorted(pack_systems), - f"--platform {platform_name} --list-systems", - ) - print( - f" WARNING: {platform_name} has no system matching " - f"{', '.join(system_filter)}. {hint}" - ) - return None - pack_systems = filtered + if system_filter and not pack_systems: + reason = system_scope_error(config, platform_name, system_filter, target_name) + print(f" WARNING: {reason}") + return None if extras_only: # The part of a split that carries the core extras no platform system # owns: the platform's own files and trees went to the other parts. @@ -1882,11 +1867,6 @@ def generate_system_pack( shutil.rmtree(scratch, ignore_errors=True) -def list_platforms(platforms_dir: str) -> list[str]: - """List available platform names from registry.""" - return list_registered_platforms(platforms_dir, include_archived=True) - - def _system_display_name(system_id: str) -> str: """Convert system ID to display name for ZIP naming.""" s = system_id.lower().replace("_", "-") @@ -1935,7 +1915,6 @@ def generate_split_packs( ) -> list[str]: """Generate split packs (one ZIP per system or manufacturer).""" config = load_platform_config(platform_name, platforms_dir) - platform_display = config.get("platform", platform_name) # The split directory carries the same tags as the packs inside it, or two # differently narrowed runs write into one another. split_tags = "".join( @@ -1951,13 +1930,19 @@ def generate_split_packs( # The parts carry the version, so the directory does too: a rescrape that # moves the version would otherwise leave the old parts beside the new # under one SHA256SUMS.txt. - version = config.get("version", config.get("dat_version", "")) - ver_tag = f"_{version.replace(' ', '')}" if version else "" - part_prefix = f"{platform_display.replace(' ', '_')}{ver_tag}{split_tags}" + # Named by the stem the full pack reads: assembled here by hand, a '/' in + # a version nested the directory where the check never looked for it. + part_prefix = ( + f"{_platform_pack_stem([platform_name], platform_name, platforms_dir)}{split_tags}" + ) split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split") os.makedirs(split_dir, exist_ok=True) - systems = config.get("systems", {}) + if emu_profiles is None: + emu_profiles = load_emulator_profiles(emulators_dir) + # The systems a core of the target runs: a part per declared system sent + # each one the target drops to a refusal naming it as unknown. + systems = platform_scope_systems(config, emu_profiles, target_cores) if group_by == "manufacturer": groups = _group_systems_by_manufacturer(systems, db, bios_dir) @@ -1966,8 +1951,6 @@ def generate_split_packs( # Pre-compute core extras once (expensive: scans 260+ emulator profiles) # then distribute per group based on emulator system overlap - if emu_profiles is None: - emu_profiles = load_emulator_profiles(emulators_dir) base_dest = config.get("base_destination", "") # Each part cuts the platform's own entries from the full profiles. The # extras are collected once here, already without the cores' game data. @@ -2373,6 +2356,46 @@ def _resolve_systems(args: argparse.Namespace, parser: argparse.ArgumentParser) parser.error(f"--system {args.system}: {exc}") +def _platforms_with_systems( + args: argparse.Namespace, + platforms: list[str], + system_filter: list[str], + emu_profiles: dict, + target_cores_cache: dict[str, set[str] | None], +) -> list[str]: + """The platforms a --system list leaves a system to pack, before any lock. + + Under --all a platform without one is out of scope, as a platform + without the target is: `--all --system sony-playstation-2` built every + PS2 pack and exited 1 on MiSTer, which has no PS2. Under --platform the + request is refused here, with its reason. + """ + kept = [] + for name in platforms: + config = load_platform_config(name, args.platforms_dir) + tc = target_cores_cache.get(name) if args.target else None + if platform_scope_systems(config, emu_profiles, tc, system_filter): + kept.append(name) + continue + if not args.all: + reason = system_scope_error(config, name, system_filter, args.target) + print(f"ERROR: {reason}", file=sys.stderr) + sys.exit(1) + declared = select_systems(config.get("systems", {}), system_filter) + reason = ( + system_scope_error(config, name, system_filter, args.target) + if declared and args.target + else f"no system {', '.join(system_filter)}" + ) + print(f"INFO: Skipping {name}: {reason}") + if not kept: + where = f" on target {args.target}" if args.target else "" + print(f"ERROR: no platform packed here runs {', '.join(system_filter)}{where}", + file=sys.stderr) + sys.exit(1) + return kept + + def _refuse_for_verify_packs(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: """What --verify-packs cannot be combined with, refused before any lock. @@ -2652,6 +2675,19 @@ def _run_verify_packs(args: argparse.Namespace) -> None: else: raise ValueError("--verify-packs requires --platform or --all") + # The target is checked as the build checks it: a mistyped one, or one + # the platform does not publish, was dropped and the full pack verified + # under it, `all OK` for a pack no build had made. + target_cores_cache: dict[str, set[str] | None] = {} + if args.target: + try: + target_cores_cache, platforms = build_target_cores_cache( + platforms, args.target, args.platforms_dir, is_all=args.all + ) + except (FileNotFoundError, ValueError) as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + all_ok = True verify_regions = getattr(args, "regions", None) verify_profiles = load_emulator_profiles(args.emulators_dir) @@ -2689,7 +2725,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None: else: print(f" {platform_name}: SKIP (no pack in {args.output_dir})") continue - tc = _target_cores_for(platform_name, args.target, args.platforms_dir) + tc = target_cores_cache.get(platform_name) # A group pack and a pack built for the platform alone can both be # present, and each one is a pack of this platform. for pack_name in found: @@ -3057,7 +3093,7 @@ def _refuse_unapplied_flags(args: argparse.Namespace, parser: argparse.ArgumentP name for name, on in ( ("--manifest-targets", args.manifest_targets), ("--verify-packs", args.verify_packs and not args.all_variants), - ("--list", args.list), + ("--list-platforms", args.list), ("--list-emulators", args.list_emulators), ("--list-systems", args.list_systems), ("--list-targets", args.list_targets), @@ -3083,7 +3119,7 @@ QUICK_MODE_FLAGS: dict[str, set[str]] = { }, # One manifest per hardware target, from the target files themselves. "--manifest-targets": set(), - "--list": set(), + "--list-platforms": set(), "--list-emulators": set(), "--list-systems": {"--platform"}, "--list-targets": {"--platform"}, @@ -3112,7 +3148,7 @@ def _given_flags(args: argparse.Namespace) -> set[str]: "--target": args.target, "--region": args.region, "--one-per-slot": args.one_per_slot, - "--list": args.list, + "--list-platforms": args.list, "--list-emulators": args.list_emulators, "--list-systems": args.list_systems, "--list-targets": args.list_targets, @@ -3209,7 +3245,11 @@ def _main(held: contextlib.ExitStack) -> None: action="store_true", help="Pack the data directories as cached, still fetching release assets", ) - parser.add_argument("--list", action="store_true", help="List available platforms") + # install.py's name. --list stays for the scripts that already type it. + parser.add_argument( + "--list-platforms", "--list", dest="list", action="store_true", + help="List available platforms", + ) parser.add_argument( "--required-only", action="store_true", @@ -3311,14 +3351,7 @@ def _main(held: contextlib.ExitStack) -> None: ) return if args.list: - # Every platform builds with --platform. --all builds the release - # packs only, the others are served by the installer. - active = set(list_registered_platforms(args.platforms_dir)) - released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True)) - for p in list_platforms(args.platforms_dir): - marks = [m for m, on in (("archived", p not in active), - ("installer only", p not in released)) if on] - print(f"{p} ({', '.join(marks)})" if marks else p) + print_platform_listing(args.platforms_dir) return if args.list_emulators: list_emulator_profiles(args.emulators_dir) @@ -3464,6 +3497,28 @@ def _main(held: contextlib.ExitStack) -> None: parser.error("Specify --platform or --all") return + # A mistyped target is refused before the output lock and the data + # directory refresh: checked after them, `--target swich` waited on a + # pipeline's caches before saying the target does not exist. + target_cores_cache: dict[str, set[str] | None] = {} + if args.target: + try: + target_cores_cache, platforms = build_target_cores_cache( + platforms, + args.target, + args.platforms_dir, + is_all=args.all, + ) + except (FileNotFoundError, ValueError) as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(1) + + emu_profiles = load_emulator_profiles(args.emulators_dir) + if system_filter: + platforms = _platforms_with_systems( + args, platforms, system_filter, emu_profiles, target_cores_cache + ) + data_registry = load_data_dir_registry(args.platforms_dir) # The output directory is claimed before the data directory refresh: a # refresh waits for every run that holds a cache, so a second build @@ -3483,21 +3538,6 @@ def _main(held: contextlib.ExitStack) -> None: if data_registry: hold_data_dirs(held, data_registry) - emu_profiles = load_emulator_profiles(args.emulators_dir) - - target_cores_cache: dict[str, set[str] | None] = {} - if args.target: - try: - target_cores_cache, platforms = build_target_cores_cache( - platforms, - args.target, - args.platforms_dir, - is_all=args.all, - ) - except (FileNotFoundError, ValueError) as e: - print(f"ERROR: {e}", file=sys.stderr) - sys.exit(1) - groups = group_identical_platforms( platforms, args.platforms_dir, target_cores_cache if args.target else None ) @@ -4877,17 +4917,12 @@ def _target_cores_for( ) -> set[str] | None: """Cores a target leaves available, or None when no target was asked for. - None means no narrowing, which is what an unfiltered pack expects. Read - the way --all builds: a platform without a target file was packed - unnarrowed under the target's name, and checking that pack must not - raise where building it did not. + Called on a platform the build already packed under the target, so the + target is known to it: a platform without one is left out of the run. """ if not target_name: return None - try: - return load_target_config(platform_name, target_name, platforms_dir) - except (FileNotFoundError, ValueError): - return None + return load_target_config(platform_name, target_name, platforms_dir) def _expected_pack_names( diff --git a/scripts/packextras.py b/scripts/packextras.py index 23d13e4b1..9bb31ada8 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -244,6 +244,7 @@ class ExtraResolver: offline: bool | None = True, *, profiles: dict, + pending_assets: bool = False, ) -> None: self.profiles = profiles self.db = db @@ -253,10 +254,13 @@ class ExtraResolver: self.validation_index = validation_index self.validation_owners = validation_owners self.offline = offline + # verify's replay: never downloads, and counts the release asset an + # online build fetches, as it does for a platform entry. + self.pending_assets = pending_assets def resolve(self, fe: dict, dest: str) -> tuple[str | None, str, str]: """(local copy or None, resolver status, the core's disagreement).""" - from packresolve import resolve_file + from packresolve import pending_release_asset, resolve_file, run_is_offline from validation import core_size_refusal, extra_alternatives, validated_choice local_path, status = resolve_file( @@ -264,6 +268,15 @@ class ExtraResolver: dest_hint=fe.get("destination", ""), data_dir_registry=self.data_registry, offline=self.offline, ) + if ( + self.pending_assets and status in ("not_found", "hash_mismatch") + and not run_is_offline() + ): + # The asset the online builder fetches in resolve_file. Its record + # answers the declaration, and there are no bytes on disk to check. + asset = pending_release_asset(fe, self.db) + if asset: + return asset, "release_asset", "" if status in ("not_found", "external", "user_provided") or not local_path: return None, status, "" local_path, disagreement = validated_choice( diff --git a/scripts/packverify.py b/scripts/packverify.py index c52f466b2..60486be1a 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -30,7 +30,7 @@ from packextras import platform_region_groups from packpaths import _flat from nativemode import reads_file_contents import region as region_mod -from packresolve import resolve_file +from packresolve import resolve_declaration, resolve_file from exporter.candidates import BIOS, EntryCategories from gamedata import without_game_data from common import resolve_platform_cores @@ -544,6 +544,21 @@ def _structural_errors(zf: zipfile.ZipFile, zip_set: set) -> list[str]: return errors +def _shipped_copy( + entry: dict, db: dict, data_registry: dict | None, mode: str, hash_rule: str, +) -> str | None: + """The file the builder packs for a declaration, or None. + + The verdict verify applies before it lets a declaration hold its path. + """ + local, status = resolve_declaration( + entry, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True, + ) + if local and baseline_verdict(entry, local, status, mode, hash_rule)[1] is None: + return local + return None + + def _builder_extras( config: dict, emulators_dir: str, @@ -563,15 +578,17 @@ def _builder_extras( """{full destination: local copy} of each core extra the builder writes. The platform's paths come first, as in the builder: a destination the - pack holds whose declarations resolve to a file, or that the platform - keeps for a user_provided or external entry. One whose declarations - resolve to nothing is left free, since the builder then places a core's - file there. + pack holds whose declarations resolve to a file it ships, or that the + platform keeps for a user_provided or external entry. One whose + declarations resolve to nothing, or to a file the frontend's hash + refuses, is left free, since the builder then places a core's file + there: taken by the refused file, that core file was never compared. """ from packextras import ExtraResolver, hidden_extras - from packresolve import resolve_declaration from validation import _build_validation_index, destination_owners + mode = config.get("verification_mode", "existence") + hash_rule = config.get("hash_rule", "") shipped: dict[str, str | None] = {} for expected, declarations in baseline_groups.items(): if expected not in zip_set and expected.lower() not in zip_lower: @@ -587,9 +604,7 @@ def _builder_extras( local = next( ( found for fe in declarations - if (found := resolve_declaration( - fe, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True, - )[0]) + if (found := _shipped_copy(fe, db, data_registry, mode, hash_rule)) ), None, ) diff --git a/scripts/slots.py b/scripts/slots.py index 5002ce86b..3680f7c69 100644 --- a/scripts/slots.py +++ b/scripts/slots.py @@ -100,6 +100,20 @@ class Conflict: def emulators(self) -> list[str]: return sorted({c.emulator for c in self.profile_claims if c.emulator}) + def files_read(self) -> list[tuple[str | None, list[str], str]]: + """Each file the profiles want here, with the emulators reading it + and the proof of the first claim on it. + + One file stood for every claim: Recalbox's neocdz.zip line gave + geolith the archive fbneo reads, not the one geolith pins. + """ + files: dict[str | None, tuple[list[str], str]] = {} + for claim in self.profile_claims: + emulators, status = files.setdefault(claim.local_path, ([], claim.status)) + if claim.emulator and claim.emulator not in emulators: + emulators.append(claim.emulator) + return [(path, sorted(emus), status) for path, (emus, status) in files.items()] + def _normalize(destination: str, fold: bool = True) -> str: """Comparable form of a destination, since layers write it differently. @@ -661,11 +675,13 @@ def format_decision(decision: Decision) -> str: f"{conflict.destination}: serve {decision.winner.local_path}, " "the pack answers to the emulator" ) - expected = ", ".join(conflict.emulators) or "the profile" + rejected = ", ".join( + f"{path}, the file {', '.join(emulators) or 'the profile'} reads" + for path, emulators, _status in conflict.files_read() + ) return ( f"{conflict.destination}: keep {decision.winner.local_path}, the frontend " - f"verifies content and would reject " - f"{conflict.profile_claims[0].local_path}, the file {expected} reads" + f"verifies content and would reject {rejected}" ) @@ -996,8 +1012,10 @@ def main() -> int: "ships": c.platform_claim.local_path, "ships_evidence": c.platform_claim.status, "emulators": c.emulators, - "expected": c.profile_claims[0].local_path, - "expected_evidence": c.profile_claims[0].status, + "expected": [ + {"file": path, "evidence": status, "emulators": emulators} + for path, emulators, status in c.files_read() + ], } for c in conflicts ] diff --git a/scripts/verify.py b/scripts/verify.py index 3c6fe1720..67aef9f42 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -29,6 +29,7 @@ import contextlib import json import os import sys +from collections import Counter from collections.abc import Mapping from pathlib import Path @@ -54,7 +55,6 @@ from common import ( compute_hashes, expand_directory_entries, platform_declarations, - filter_systems_by_target, group_identical_platforms, list_emulator_profiles, list_system_ids, @@ -63,7 +63,10 @@ from common import ( load_emulator_profiles, load_platform_config, resolve_system_arg, + print_platform_listing, select_systems, + platform_scope_systems, + system_scope_error, md5sum, require_yaml, resolve_local_file, @@ -1217,16 +1220,7 @@ def verify_platform( validation_index = _build_validation_index(platform_profiles) validation_owners = destination_owners(platform_profiles) - if not target_cores: - plat_cores = None - verify_systems = filter_systems_by_target( - config.get("systems", {}), - profiles, - target_cores, - platform_cores=plat_cores, - ) - if system_filter: - verify_systems = select_systems(verify_systems, system_filter) + verify_systems = platform_scope_systems(config, profiles, target_cores, system_filter) details = [] file_status: dict[str, str] = {} @@ -1315,8 +1309,12 @@ def verify_platform( "release_asset": local_path, "reason": "release asset, fetched when the pack is built", } + # Written at its path like any held file: keyed to None, it + # stayed out of pack_destinations and step 5 read the pack's + # copy as a file only the pack holds. packed_order.append(( - sys_id, f"{base_dest}/{destination}" if base_dest else destination, None, + sys_id, f"{base_dest}/{destination}" if base_dest else destination, + local_path, )) shipped_keys.add(_dest_key(file_entry, sys_id)) local_path = None @@ -1465,7 +1463,7 @@ def verify_platform( region_drops, report_for_builder, ExtraResolver( db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index, - validation_owners, profiles=profiles, + validation_owners, profiles=profiles, pending_assets=True, ), placed_kind, system_filter, @@ -1746,13 +1744,20 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) -> print(f" {line}") -def _print_undeclared_entry(u: dict, prefix: str, verbose: bool) -> None: - """Print a single undeclared file entry with its validation checks.""" +def _print_undeclared_entry( + u: dict, prefix: str, verbose: bool, repeated: frozenset[str] = frozenset() +) -> None: + """Print a single undeclared file entry with its validation checks. + + A name in ``repeated`` is read at more than one path, and the line names + the path: two identical lines told no one there were two slots to fill. + """ + name = u["path"] if u["name"] in repeated and u.get("path") else u["name"] arc_count = u.get("archive_file_count") if arc_count: - name_label = f"{u['name']} ({arc_count} file{'s' if arc_count != 1 else ''})" + name_label = f"{name} ({arc_count} file{'s' if arc_count != 1 else ''})" else: - name_label = u["name"] + name_label = name print(f" {prefix}: {u['emulator']} needs {name_label}") checks = u.get("checks", []) if checks: @@ -1800,6 +1805,25 @@ def _core_required_missing(bios_files: list[dict]) -> list[dict]: ] +def _repeated_names(entries: list[dict]) -> frozenset[str]: + """Names more than one of ``entries`` carries.""" + counts = Counter(u["name"] for u in entries) + return frozenset(name for name, count in counts.items() if count > 1) + + +def _omission_reasons(entries: list[dict]) -> str: + """Each reason a held file was left out, with its count when several. + + Printing the first entry's reason for the whole count named one cause + for 412 RomM files when 20 carried it. + """ + counts = Counter(u.get("omitted", "not placed") for u in entries) + if len(counts) == 1: + return next(iter(counts)) + ranked = sorted(counts.items(), key=lambda item: (-item[1], item[0])) + return ", ".join(f"{count} {reason}" for reason, count in ranked) + + def _print_undeclared_section(result: dict, verbose: bool) -> None: """Print cross-reference section for undeclared files used by cores.""" undeclared = result.get("undeclared_files", []) @@ -1839,19 +1863,22 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, " f"{core_missing_opt} optional missing{listed}" ) + # The missing files follow the line that counts them: printed after the + # held-but-not-packed line, they read as files installed with their emulator. + repeated = _repeated_names(req_not_in_repo + req_hle_not_in_repo + opt_not_in_repo) + for u in req_not_in_repo: + _print_undeclared_entry(u, "MISSING (required)", verbose, repeated) + for u in req_hle_not_in_repo: + _print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose, repeated) + if verbose: + for u in opt_not_in_repo: + _print_undeclared_entry(u, "MISSING (optional)", verbose, repeated) if bundled: names = ", ".join(sorted({u["name"] for u in bundled})) print(f" Bundled with their emulator, not collected: {names}") if unplaced: - print(f" Core files held but not packed: {len(unplaced)} ({unplaced[0]['omitted']})") - - for u in req_not_in_repo: - _print_undeclared_entry(u, "MISSING (required)", verbose) - for u in req_hle_not_in_repo: - _print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose) - if verbose: - for u in opt_not_in_repo: - _print_undeclared_entry(u, "MISSING (optional)", verbose) + print(f" Core files held but not packed: {len(unplaced)} " + f"({_omission_reasons(unplaced)})") if game_data: gd_missing = [u for u in game_data if not u["in_repo"]] @@ -1864,12 +1891,13 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None: listed = "" if verbose or not gd_missing else " (--verbose lists them)" print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}") if verbose: + repeated = _repeated_names(gd_missing) for u in gd_missing: - _print_undeclared_entry(u, "MISSING (game data)", verbose) + _print_undeclared_entry(u, "MISSING (game data)", verbose, repeated) if gd_unplaced: print( f" Game data held but not packed: {len(gd_unplaced)} " - f"({gd_unplaced[0].get('omitted', 'not placed')})" + f"({_omission_reasons(gd_unplaced)})" ) @@ -2597,6 +2625,7 @@ def _refuse_listing_narrowings( """ listing = next( (flag for flag, on in ( + ("--list-platforms", args.list_platforms), ("--list-emulators", args.list_emulators), ("--list-systems", args.list_systems), ("--list-targets", args.list_targets), @@ -2605,7 +2634,7 @@ def _refuse_listing_narrowings( ) if not listing: return - reads_platform = listing != "--list-emulators" + reads_platform = listing not in ("--list-emulators", "--list-platforms") for flag, on in ( ("--platform", args.platform and not reads_platform), ("--all", args.all), @@ -2655,7 +2684,9 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser def _run_listing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: - if args.list_emulators: + if args.list_platforms: + print_platform_listing(args.platforms_dir) + elif args.list_emulators: list_emulator_profiles(args.emulators_dir) elif args.list_systems and args.platform: list_platform_system_ids(args.platform, args.platforms_dir) @@ -2694,6 +2725,9 @@ def _main(held: contextlib.ExitStack) -> None: parser.add_argument( "--list-systems", action="store_true", help="List available systems" ) + parser.add_argument( + "--list-platforms", action="store_true", help="List available platforms" + ) parser.add_argument( "--include-archived", action="store_true", help="With --all, also verify the archived platforms (RetroPie)", @@ -2767,7 +2801,7 @@ def _main(held: contextlib.ExitStack) -> None: parser.error(str(exc)) _refuse_listing_narrowings(args, parser) - if args.list_emulators or args.list_systems or args.list_targets: + if args.list_platforms or args.list_emulators or args.list_systems or args.list_targets: _run_listing(args, parser) return @@ -2853,6 +2887,18 @@ def _main(held: contextlib.ExitStack) -> None: except (FileNotFoundError, ValueError) as e: print(f"ERROR: {e}", file=sys.stderr) sys.exit(1) + # A system the target leaves no core for was reported 0/0 with exit 0, + # where generate_pack refuses the same request. + if platform_systems: + config = load_platform_config(args.platform, args.platforms_dir) + if not platform_scope_systems( + config, emu_profiles, target_cores_cache.get(args.platform), platform_systems + ): + reason = system_scope_error( + config, args.platform, platform_systems, args.target + ) + print(f"ERROR: {reason}", file=sys.stderr) + sys.exit(1) # Group identical platforms (same function as generate_pack) groups = group_identical_platforms( @@ -2961,8 +3007,7 @@ def _overall_summary(group_results: list[tuple[dict, list[str]]]) -> str: platforms = sum(len(group) for _, group in group_results) def where(counts: dict[str, int]) -> str: - named = [f"{name} {count}" if count > 1 else name - for name, count in counts.items() if count] + named = [f"{name} {count}" for name, count in counts.items() if count] return f" ({', '.join(named)})" if named else "" parts = [] diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 4f18a8ca8..3ece3f0a6 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -4830,7 +4830,7 @@ class VerifyAllSummaryNamesWhatIsWrong(unittest.TestCase): summary = verify._overall_summary( [(clean, ["RetroArch"]), (warned, ["RetroDECK"]), (core_miss, ["ROCKNIX"])] ) - self.assertIn("1 warning (RetroDECK)", summary) + self.assertIn("1 warning (RetroDECK 1)", summary) self.assertIn("ROCKNIX: n64.psh", summary) self.assertIn("0 critical,", summary) diff --git a/tests/test_pack_core_extras.py b/tests/test_pack_core_extras.py index d997b8b97..82c10eed9 100644 --- a/tests/test_pack_core_extras.py +++ b/tests/test_pack_core_extras.py @@ -87,6 +87,59 @@ class CoreFilesAsTheBuilderPlacesThem(unittest.TestCase): ) +class ARefusedPinLeavesItsPathToTheCore(unittest.TestCase): + """EmuELEC pins naomi.zip by an md5 the collection does not hold: the + builder refuses the held copy and ships the core's file at that path. + The check marked the path taken by the refused platform file, so the + core file the pack carries there was never compared.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") + self.addCleanup(tmp.cleanup) + previous = os.getcwd() + os.chdir(tmp.name) + self.addCleanup(os.chdir, previous) + files: dict = {} + by_name: dict = {} + _held(files, by_name, "bios/dc/naomi.zip", b"the core's naomi") + self.db = {"files": files, "indexes": { + "by_name": by_name, "by_md5": {e["md5"]: s for s, e in files.items()}, + "by_path_suffix": {}, "by_crc32": {}, "by_sha256": {}}} + Path("platforms").mkdir() + config = {"platform": "P", "verification_mode": "md5", "cores": ["flycast"], + "systems": {"dc": {"files": [{ + "name": "naomi.zip", "destination": "dc/naomi.zip", + "md5": hashlib.md5(b"a bundle nobody holds").hexdigest(), + }]}}} + Path("platforms/p.yml").write_text(yaml.safe_dump(config), encoding="utf-8") + Path("emulators").mkdir() + self.profiles = {"flycast": { + "emulator": "flycast", "type": "libretro", "systems": ["dc"], "files": [ + {"name": "naomi.zip", "path": "dc/naomi.zip"}]}} + + def _check(self, member: bytes) -> tuple: + with zipfile.ZipFile("pack.zip", "w") as zf: + zf.writestr("dc/naomi.zip", member) + return packverify.verify_pack_against_platform( + "pack.zip", "p", "platforms", self.db, emulators_dir="emulators", + emu_profiles=self.profiles, + ) + + def test_the_core_file_at_the_refused_path_is_compared(self): + _ok, _c, _p, errors, *_mid, core_checked, _cp, _ex, _ab = self._check(b"other") + self.assertEqual(core_checked, 1) + self.assertTrue( + any(e.startswith("core file differs: dc/naomi.zip") for e in errors), errors + ) + + def test_the_right_core_file_passes(self): + _ok, _c, _p, errors, *_mid, core_checked, core_present, _ex, _ab = ( + self._check(b"the core's naomi") + ) + self.assertEqual((core_checked, core_present), (1, 1)) + self.assertFalse([e for e in errors if e.startswith("core")], errors) + + class NarrowedPacksAgainstTheirOwnList(unittest.TestCase): """--verify-packs after --all-variants said "hashes checked during the build" for the five narrowed packs and compared none of them: a narrowed diff --git a/tests/test_slots.py b/tests/test_slots.py index 864dede99..d4dabab91 100644 --- a/tests/test_slots.py +++ b/tests/test_slots.py @@ -420,6 +420,24 @@ class TestArbitration(unittest.TestCase): served = slots.format_decision(slots.arbitrate(self._conflict(), "existence")) self.assertIn("both are satisfied", served) + def test_each_emulator_is_given_the_file_it_reads(self): + """Recalbox's neocdz.zip line said geolith reads the archive fbneo + reads, while geolith pins another one.""" + conflict = self._conflict() + conflict.profile_claims = [ + slots.Claim("profile", "neocdz.zip", "neocdz.zip", emulator="fbneo", + local_path="a/neocdz.zip.ba20a7be", status="members_held"), + slots.Claim("profile", "neocdz.zip", "neocdz.zip", emulator="geolith", + local_path="b/neocdz.zip.712816a3", status="md5_exact"), + ] + kept = slots.format_decision(slots.arbitrate(conflict, "md5")) + self.assertIn("a/neocdz.zip.ba20a7be, the file fbneo reads", kept) + self.assertIn("b/neocdz.zip.712816a3, the file geolith reads", kept) + self.assertEqual(conflict.files_read(), [ + ("a/neocdz.zip.ba20a7be", ["fbneo"], "members_held"), + ("b/neocdz.zip.712816a3", ["geolith"], "md5_exact"), + ]) + class TheStrongestProofTakesAnUndeclaredPath(unittest.TestCase): """No platform declares RetroDECK's hiscore.dat. fbalpha2012 asks for it diff --git a/tests/test_target_verify.py b/tests/test_target_verify.py index d9f895ec5..ee42f854b 100644 --- a/tests/test_target_verify.py +++ b/tests/test_target_verify.py @@ -4,11 +4,13 @@ under the target's name (Recalbox, Lakka, BizHawk, MiSTer, ROCKNIX). The conformance check read the same target without the --all tolerance, raised FileNotFoundError on the first of those packs and stopped the run in a -traceback after every pack had been written. +traceback after every pack had been written. Such a platform is now left +out of a targeted --all run, and the check refuses it as the build does. """ from __future__ import annotations +import os import sys import contextlib import io @@ -31,7 +33,10 @@ class CheckReadsTheTargetLikeTheBuild(unittest.TestCase): contextlib.redirect_stdout(io.StringIO()): with self.assertRaises(ValueError): build_target_cores_cache(["recalbox"], "x86_64", platforms, is_all=True) - self.assertIsNone(_target_cores_for("recalbox", "x86_64", platforms)) + # The build never packs such a platform under the target, so the + # check refuses it too instead of reading the full pack. + with self.assertRaises(FileNotFoundError): + _target_cores_for("recalbox", "x86_64", platforms) def test_a_platform_with_a_target_file_is_narrowed(self): cores = _target_cores_for("retroarch", "switch", str(REPO_ROOT / "platforms")) @@ -54,5 +59,167 @@ class ANameClaimsOnlyANarrowingThatHappened(unittest.TestCase): ) +class AMistypedTargetIsRefusedFirst(unittest.TestCase): + """`--platform retroarch --target swich` took the dist lock and waited on + the data directory refresh, 22 s here and hours behind a pipeline, before + saying the target does not exist.""" + + def test_no_lock_and_no_refresh_before_the_refusal(self): + from unittest import mock + + import generate_pack + + def touched(*_a, **_kw): + raise AssertionError("lock or refresh taken before the target check") + + argv = ["generate_pack.py", "--platform", "retroarch", "--target", "swich", + "--output-dir", str(REPO_ROOT / "tmp" / "never-written")] + err = io.StringIO() + with mock.patch.object(sys, "argv", argv), \ + mock.patch.object(generate_pack, "_pack_output_lock", touched), \ + mock.patch.object(generate_pack, "hold_data_dirs", touched), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(err), \ + self.assertRaises(SystemExit) as ctx: + generate_pack.main() + self.assertEqual(ctx.exception.code, 1) + self.assertIn("Unknown target 'swich'", err.getvalue()) + + +class ASystemTheTargetDropsIsNamedAsSuch(unittest.TestCase): + """`--platform retroarch --system sony-playstation-2 --target switch`: + verify printed 0/0 with exit 0 under --strict, and generate_pack said + RetroArch has no such system and suggested sony-playstation.""" + + PLATFORMS = str(REPO_ROOT / "platforms") + + def test_both_tools_read_one_scope(self): + from common import ( + load_emulator_profiles, load_platform_config, platform_scope_systems, + ) + + config = load_platform_config("retroarch", self.PLATFORMS) + profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + switch = _target_cores_for("retroarch", "switch", self.PLATFORMS) + wanted = ["sony-playstation-2"] + self.assertEqual(platform_scope_systems(config, profiles, switch, wanted), {}) + self.assertEqual(list(platform_scope_systems(config, profiles, None, wanted)), wanted) + + def test_the_message_names_the_target(self): + from common import system_scope_error + + config = {"systems": {"sony-playstation-2": {}, "sony-playstation": {}}} + self.assertEqual( + system_scope_error(config, "retroarch", ["sony-playstation-2"], "switch"), + "no core retroarch runs on target switch emulates sony-playstation-2", + ) + undeclared = system_scope_error(config, "retroarch", ["nec-pc-98"], "switch") + self.assertTrue(undeclared.startswith("retroarch has no system matching nec-pc-98.")) + + def test_verify_refuses_it(self): + import subprocess + + proc = subprocess.run( + [sys.executable, "scripts/verify.py", "--platform", "retroarch", + "--system", "sony-playstation-2", "--target", "switch", "--strict"], + capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=600, check=False, + ) + self.assertEqual(proc.returncode, 1, proc.stdout[-300:]) + self.assertIn("no core retroarch runs on target switch emulates sony-playstation-2", + proc.stderr) + + +class ASplitBuildsOnlyTheTargetsSystems(unittest.TestCase): + """`--split --target switch` asked for a part per declared system, and + each one the target drops ended on 'retroarch has no system matching'.""" + + def test_no_part_is_asked_for_a_dropped_system(self): + from unittest import mock + + import generate_pack + from common import load_emulator_profiles + + platforms = str(REPO_ROOT / "platforms") + profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + asked: list[list[str]] = [] + + def record(*_a, **kw): + if kw.get("system_filter"): + asked.append(kw["system_filter"]) + return None + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \ + mock.patch.object(generate_pack, "generate_pack", record), \ + mock.patch.object(generate_pack, "_collect_emulator_extras", + return_value=[]): + generate_pack.generate_split_packs( + "retroarch", platforms, {"files": {}}, "bios", out, + emu_profiles=profiles, + target_cores=_target_cores_for("retroarch", "switch", platforms), + target_name="nintendo-switch", + ) + systems = {sid for ids in asked for sid in ids} + self.assertTrue(systems) + self.assertNotIn("sony-playstation-2", systems) + self.assertIn("sony-playstation", systems) + + +class APlatformWithoutTheSystemIsOutOfScope(unittest.TestCase): + """`--all --system sony-playstation-2` built the five PS2 packs and + exited 1 on 'no pack for misterfpga', which has no PS2.""" + + def _scope(self, **args): + import argparse + + import generate_pack + from common import load_emulator_profiles + + values = {"platforms_dir": str(REPO_ROOT / "platforms"), "target": None, + "all": True} + values.update(args) + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + kept = generate_pack._platforms_with_systems( + argparse.Namespace(**values), ["retroarch", "misterfpga"], + ["sony-playstation-2"], + load_emulator_profiles(str(REPO_ROOT / "emulators")), {}, + ) + return kept, out.getvalue() + + def test_all_skips_it(self): + kept, out = self._scope() + self.assertEqual(kept, ["retroarch"]) + self.assertIn("INFO: Skipping misterfpga: no system sony-playstation-2", out) + + def test_a_named_platform_is_refused(self): + with self.assertRaises(SystemExit) as ctx: + self._scope(all=False) + self.assertEqual(ctx.exception.code, 1) + + +class TheSplitDirectoryReadsThePackStem(unittest.TestCase): + """The split prefix was assembled by hand: a version 'release/1.22' made + 'RetroArch_release/1.22_Split', a nested directory the check never + found, beside a full pack named 'RetroArch_release-1.22_BIOS_Pack.zip'.""" + + def test_a_reserved_character_is_made_safe_as_in_the_full_pack(self): + from unittest import mock + + import generate_pack + + config = {"platform": "Foo: Bar", "version": "release/1.22", "systems": {}} + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \ + mock.patch.object(generate_pack, "load_platform_config", + return_value=config), \ + mock.patch.object(generate_pack, "_collect_emulator_extras", + return_value=[]): + generate_pack.generate_split_packs( + "foo", "platforms", {"files": {}}, "bios", out, emu_profiles={}, + ) + stem = generate_pack._platform_pack_stem(["foo"], "foo", "platforms") + self.assertEqual(os.listdir(out), [f"{stem}_Split"]) + self.assertEqual(stem, "Foo-_Bar_release-1.22") + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_tool_messages.py b/tests/test_tool_messages.py index adfcdd59d..3d782366f 100644 --- a/tests/test_tool_messages.py +++ b/tests/test_tool_messages.py @@ -132,7 +132,7 @@ class TheVerifySummaryCountsDiscrepancies(unittest.TestCase): ({"severity_counts": {}, "details": details}, ["RetroArch"]), ({"severity_counts": {}, "details": []}, ["Batocera"]), ]) - self.assertIn("1 discrepancy (RetroArch)", summary) + self.assertIn("1 discrepancy (RetroArch 1)", summary) def test_a_shared_destination_names_the_path_and_both_sides(self): result = {"name": "disk.rom", "status": verify.Status.UNTESTED} @@ -145,6 +145,121 @@ class TheVerifySummaryCountsDiscrepancies(unittest.TestCase): self.assertIn(part, message) +class EveryPlatformInTheSummaryCarriesItsCount(unittest.TestCase): + """'17 discrepancies (Lakka 5, RetroArch 5, RetroBat 2, RetroDECK 4, + ROCKNIX)': the one platform with a single entry had no number, and the + named counts did not add up to the total.""" + + def test_a_count_of_one_is_written(self): + one = {"severity_counts": {verify.Severity.WARNING: 1}, "details": []} + two = {"severity_counts": {verify.Severity.WARNING: 2}, "details": []} + summary = verify._overall_summary([(two, ["Lakka"]), (one, ["ROCKNIX"])]) + self.assertIn("3 warning (Lakka 2, ROCKNIX 1)", summary) + + +class HeldButNotPackedNamesEachReason(unittest.TestCase): + """'Core files held but not packed: 16 (a packed file occupies its + directory)' on Lakka, where 14 of the 16 are installed with their + emulator: the first entry's reason stood for the whole count.""" + + @staticmethod + def _entry(name: str, reason: str, category: str = "bios") -> dict: + return {"name": name, "emulator": "x", "required": True, "in_repo": True, + "in_pack": False, "omitted": reason, "category": category} + + def _printed(self, entries: list[dict]) -> str: + out = io.StringIO() + with contextlib.redirect_stdout(out): + verify._print_undeclared_section({"undeclared_files": entries}, False) + return out.getvalue() + + def test_each_reason_carries_its_count(self): + entries = [self._entry("a.rom", "a packed file occupies its directory")] + entries += [self._entry(f"{i}.rom", "installed with its emulator") for i in range(3)] + entries += [self._entry(f"g{i}", "installed with its emulator", "game_data") + for i in range(2)] + entries.append(self._entry("g9", "no platform slug for its system", "game_data")) + text = self._printed(entries) + self.assertIn( + "Core files held but not packed: 4 (3 installed with its emulator, " + "1 a packed file occupies its directory)", text) + self.assertIn( + "Game data held but not packed: 3 (2 installed with its emulator, " + "1 no platform slug for its system)", text) + + def test_a_missing_file_follows_the_line_that_counts_it(self): + """Indented under 'held but not packed: 102 (installed with its + emulator)', Batocera's one missing sfiii2a.zip read as one of them.""" + missing = {"name": "sfiii2a.zip", "emulator": "FinalBurn Neo", "required": True, + "in_repo": False, "category": "bios"} + lines = self._printed( + [missing, self._entry("a.rom", "installed with its emulator")] + ).splitlines() + self.assertTrue(lines[0].startswith(" Core files: "), lines) + self.assertIn("MISSING (required): FinalBurn Neo needs sfiii2a.zip", lines[1]) + self.assertTrue(lines[2].startswith(" Core files held but not packed: "), lines) + + def test_one_name_read_at_two_paths_prints_both_paths(self): + """MyMinUI printed 'FinalBurn Neo needs sfiii2a.zip' twice, word for + word, for Bios/FBN/ and Bios/NG/.""" + base = {"name": "sfiii2a.zip", "emulator": "FinalBurn Neo", "required": True, + "in_repo": False, "category": "bios"} + lone = dict(base, name="other.zip", path="Bios/FBN/other.zip") + text = self._printed([dict(base, path="Bios/FBN/sfiii2a.zip"), + dict(base, path="Bios/NG/sfiii2a.zip"), lone]) + self.assertIn("FinalBurn Neo needs Bios/FBN/sfiii2a.zip", text) + self.assertIn("FinalBurn Neo needs Bios/NG/sfiii2a.zip", text) + self.assertIn("FinalBurn Neo needs other.zip\n", text) + + def test_a_single_reason_reads_as_before(self): + text = self._printed([self._entry("a.rom", "installed with its emulator")] * 2) + self.assertIn("Core files held but not packed: 2 (installed with its emulator)", text) + + +class APlatformWithoutTargetsSaysSo(unittest.TestCase): + """`--platform misterfpga --target switch` printed 'No target config for + platform 'misterfpga': platforms/targets/misterfpga.yml', which reads as + a file missing from the repository.""" + + def test_the_message_names_the_reason_and_the_platforms_with_targets(self): + with tempfile.TemporaryDirectory() as tmp: + (Path(tmp) / "targets").mkdir() + for name in ("retroarch", "batocera", "_overrides"): + (Path(tmp) / "targets" / f"{name}.yml").write_text("targets: {}\n") + with self.assertRaises(FileNotFoundError) as caught: + common.load_target_config("misterfpga", "switch", tmp) + self.assertEqual( + str(caught.exception), + "misterfpga has no hardware targets: drop --target " + "(platforms with targets: batocera, retroarch)", + ) + + +class OneNameListsThePlatforms(unittest.TestCase): + """install.py --list-platforms, generate_pack.py --list, and no listing + at all in verify.py: three scripts, three answers to one question.""" + + def _run(self, script: str, flag: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, f"scripts/{script}", flag], capture_output=True, text=True, + cwd=str(REPO_ROOT), timeout=120, check=False, + ) + + def test_both_tools_take_it_and_agree(self): + verify_out = self._run("verify.py", "--list-platforms") + pack_out = self._run("generate_pack.py", "--list-platforms") + self.assertEqual(verify_out.returncode, 0, verify_out.stderr) + self.assertEqual(pack_out.returncode, 0, pack_out.stderr) + self.assertEqual(verify_out.stdout, pack_out.stdout) + self.assertIn("retroarch", verify_out.stdout.split()) + self.assertIn("--list-platforms", (REPO_ROOT / "install.py").read_text()) + + def test_the_installer_name_is_the_one_documented(self): + tools = (REPO_ROOT / "wiki" / "tools.md").read_text() + self.assertIn("generate_pack.py --list-platforms", tools) + self.assertIn("verify.py --list-platforms", tools) + + class OneSpellingPerOption(unittest.TestCase): """verify.py -p batocera worked and slots.py -p batocera was refused.""" diff --git a/tests/test_verify_pack_choice.py b/tests/test_verify_pack_choice.py index 4aac344d8..4589df318 100644 --- a/tests/test_verify_pack_choice.py +++ b/tests/test_verify_pack_choice.py @@ -116,6 +116,24 @@ class VerifyPackChoice(unittest.TestCase): dict(zip((name for _p, name in checked), self.variants)), names ) + def test_a_target_the_platform_lacks_is_refused(self): + """`--platform bizhawk --target rpi4typo --verify-packs` verified the + full pack and said `all OK`, where the build refused the target.""" + import contextlib + import io + + bizhawk = self.gp._platform_pack_stem(["bizhawk"], "bizhawk", "platforms") + self._touch(f"{bizhawk}_BIOS_Pack.zip") + self._touch(f"{self.stem}_BIOS_Pack.zip") + for platform, target in (("bizhawk", "rpi4typo"), ("retroarch", "swich")): + with self.subTest(platform=platform), \ + contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit) as ctx: + self._run(self._args(platform=platform, target=target)) + self.assertEqual(ctx.exception.code, 1) + self.assertEqual(self.variants, []) + def test_the_run_ends_on_a_summary(self): """A failing pack was named only above its own error lines, and the log ended on the last platform's OK line.""" diff --git a/tests/test_verify_release_asset.py b/tests/test_verify_release_asset.py index f8c69c3b4..58150b375 100644 --- a/tests/test_verify_release_asset.py +++ b/tests/test_verify_release_asset.py @@ -89,6 +89,12 @@ class ReleaseAssetNotCached(unittest.TestCase): self.assertEqual(entry["status"], Status.OK) self.assertEqual(entry["release_asset"], REGISTERED[0]) + def test_its_path_is_a_pack_destination(self): + """The pack holds the fetched file at its path: absent from + pack_destinations, step 5 reported it 'pack only'.""" + report = verify_platform(self.config, self.db, "emulators", {}) + self.assertIn("big.bin", report["pack_destinations"]) + def test_offline_counts_it_missing(self): packresolve.set_offline(True) self.assertEqual(self._entry()["status"], Status.MISSING) @@ -117,6 +123,59 @@ class ReleaseAssetNotCachedMd5(ReleaseAssetNotCached): mode = "md5" +class CoreExtraReleaseAssetNotCached(ReleaseAssetNotCached): + """The same asset read by a core and declared by no platform entry: the + builder fetched and packed it online, verify's replay of the extras + resolved it offline and left its path out of the pack.""" + + def setUp(self): + super().setUp() + self.config["systems"]["sys"]["files"] = [] + self.config["cores"] = ["bigcore"] + Path("platforms/big.yml").write_text(yaml.dump(self.config)) + self.profiles = {"bigcore": { + "emulator": "Big Core", "type": "libretro", "systems": ["sys"], + "files": [{"name": "big.bin", "required": True, "sha1": SHA1}], + }} + Path("emulators/bigcore.yml").write_text(yaml.dump(self.profiles["bigcore"])) + common._platform_config_cache.clear() + common._emulator_profiles_cache.clear() + + def _report(self) -> dict: + return verify_platform(self.config, self.db, "emulators", self.profiles) + + def test_counted_as_fetched_when_the_pack_is_built(self): + report = self._report() + self.assertIn("big.bin", report["pack_destinations"]) + self.assertEqual(report["packed_extras"]["bios"], 1) + + def test_its_path_is_a_pack_destination(self): + cache = Path("cache/big.bin") + cache.parent.mkdir() + cache.write_bytes(PAYLOAD) + with mock.patch.object( + packresolve, "fetch_large_file", return_value=str(cache) + ): + zip_path = builder.generate_pack( + "big", "platforms", self.db, "bios", "out", + emulators_dir="emulators", emu_profiles=self.profiles, offline=False, + ) + with zipfile.ZipFile(zip_path) as zf: + packed = {n for n in zf.namelist() if not n.endswith("/")} + self.assertIn("big.bin", packed) + self.assertIn("big.bin", self._report()["pack_destinations"]) + + def test_offline_counts_it_missing(self): + packresolve.set_offline(True) + self.assertNotIn("big.bin", self._report()["pack_destinations"]) + + def test_an_asset_with_other_content_is_not_counted(self): + self.profiles["bigcore"]["files"][0]["sha1"] = "0" * 40 + self.assertNotIn("big.bin", self._report()["pack_destinations"]) + + test_the_online_build_packs_it = None + + class OfflinePipelineVerifiesOffline(unittest.TestCase): """The offline pipeline builds packs without release assets: its verify step has to count them the same way, or step 5 compares two answers.""" diff --git a/wiki/reading-reports.md b/wiki/reading-reports.md index 159fef7b7..c6ce6e212 100644 --- a/wiki/reading-reports.md +++ b/wiki/reading-reports.md @@ -117,7 +117,8 @@ RetroDECK: 2007/2008 OK, 1 untested [md5] The run prints more `DISCREPANCY` lines than this excerpt shows. The core and game data lines count what the emulator cores add beyond the platform's -own list. A file held but not packed is left out for the reason in parentheses. +own list. A file held but not packed is left out for the reason in parentheses, each +reason with its count when the files have several. `--verbose` also lists the passing entries that an emulator validates, with the check and the source lines behind it: diff --git a/wiki/tools.md b/wiki/tools.md index 1aa6aef87..ed20d0bff 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -134,6 +134,7 @@ python scripts/verify.py --platform recalbox --region us # regional priority l python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build +python scripts/verify.py --list-platforms # list all platforms python scripts/verify.py --list-emulators # list all emulators python scripts/verify.py --list-systems # list all systems python scripts/verify.py --platform retroarch --list-targets # list available targets @@ -208,7 +209,7 @@ python scripts/generate_pack.py --all --all-variants --output-dir dist/ # all 6 python scripts/generate_pack.py --all --all-variants --verify-packs --output-dir dist/ # Listing and integrity -python scripts/generate_pack.py --list # list available platforms +python scripts/generate_pack.py --list-platforms # list available platforms python scripts/generate_pack.py --list-emulators python scripts/generate_pack.py --list-systems python scripts/generate_pack.py --platform retroarch --list-targets