chore: merge audit batch, verify and pack

This commit is contained in:
Abdessamad Derraz committed 2026-11-05 13:16:03 +01:00
commit 3f50163b0a
15 files changed
+748 -127

No files matched your search

+64 -1
View File
@@ -664,8 +664,15 @@ def load_target_config(
targets_dir = os.path.join(platforms_dir, "targets") targets_dir = os.path.join(platforms_dir, "targets")
target_file = os.path.join(targets_dir, f"{platform_name}.yml") target_file = os.path.join(targets_dir, f"{platform_name}.yml")
if not os.path.exists(target_file): if not os.path.exists(target_file):
# The expected state for a platform that ships one build, said as
# such: the file path read like a file missing from the repository.
with_targets = sorted(
Path(f).stem for f in os.listdir(targets_dir)
if f.endswith(".yml") and not f.startswith("_")
)
raise FileNotFoundError( raise FileNotFoundError(
f"No target config for platform '{platform_name}': {target_file}" f"{platform_name} has no hardware targets: drop --target "
f"(platforms with targets: {', '.join(with_targets)})"
) )
with open(target_file) as f: with open(target_file) as f:
data = yaml_load(f) or {} data = yaml_load(f) or {}
@@ -2986,6 +2993,46 @@ def filter_systems_by_target(
} }
def platform_scope_systems(
config: dict,
profiles: dict[str, dict],
target_cores: set[str] | None,
system_filter: list[str] | None = None,
) -> dict[str, dict]:
"""The systems a pack or a report of a platform covers.
Those a core of the target runs, then the ones a --system list keeps.
verify.py and generate_pack.py read this one function.
"""
plat_cores = resolve_platform_cores(config, profiles) if target_cores else None
systems = filter_systems_by_target(
config.get("systems", {}), profiles, target_cores, platform_cores=plat_cores
)
return select_systems(systems, system_filter) if system_filter else systems
def system_scope_error(
config: dict, platform_name: str, system_filter: list[str], target: str | None
) -> str:
"""Why a --system list leaves a platform no system.
A system the platform declares but no core of the target runs was
reported as a system the platform lacks, with a lookalike to try and a
listing that refuses --target.
"""
systems = config.get("systems", {})
declared = select_systems(systems, system_filter)
if declared and target:
return (
f"no core {platform_name} runs on target {target} emulates "
f"{', '.join(sorted(declared))}"
)
hint = suggest_names(
system_filter, sorted(systems), f"--platform {platform_name} --list-systems"
)
return f"{platform_name} has no system matching {', '.join(system_filter)}. {hint}"
def platform_declarations( def platform_declarations(
config: dict, db: dict, *, enrich: bool = True config: dict, db: dict, *, enrich: bool = True
) -> dict[str, list[tuple[int | None, str]]]: ) -> dict[str, list[tuple[int | None, str]]]:
@@ -3145,6 +3192,22 @@ def list_system_ids(emulators_dir: str) -> None:
print(f" {spelling:35s} ({count} emulator{'s' if count > 1 else ''})") print(f" {spelling:35s} ({count} emulator{'s' if count > 1 else ''})")
def print_platform_listing(platforms_dir: str) -> None:
"""Every registered platform, one per line, marked when archived or
served by the installer only.
Every platform takes --platform. --all covers the active ones, and the
pack builder's --all the release packs. Printed by --list-platforms in
verify.py and generate_pack.py, the name install.py gives it.
"""
active = set(list_registered_platforms(platforms_dir))
released = set(list_release_pack_platforms(platforms_dir, include_archived=True))
for name in list_registered_platforms(platforms_dir, include_archived=True):
marks = [m for m, on in (("archived", name not in active),
("installer only", name not in released)) if on]
print(f"{name} ({', '.join(marks)})" if marks else name)
def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None: def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None:
"""Print system IDs from a platform's YAML config, with the platform's """Print system IDs from a platform's YAML config, with the platform's
own names for them, which --system also accepts.""" own names for them, which --system also accepts."""
+107 -72
View File
@@ -53,7 +53,10 @@ from common import (
list_platform_system_ids, list_platform_system_ids,
platform_system_names, platform_system_names,
resolve_system_arg, resolve_system_arg,
print_platform_listing,
select_systems, select_systems,
platform_scope_systems,
system_scope_error,
canonical_target_name, canonical_target_name,
platform_files_target, platform_files_target,
list_registered_platforms, list_registered_platforms,
@@ -753,31 +756,13 @@ def generate_pack(
target_cores=target_cores, source=source, target_cores=target_cores, source=source,
) )
# Filter systems by target if specified pack_systems = platform_scope_systems(
plat_cores = ( config, emu_profiles or {}, target_cores, system_filter
resolve_platform_cores(config, emu_profiles or {}) if target_cores else None
) )
pack_systems = filter_systems_by_target( if system_filter and not pack_systems:
config.get("systems", {}), reason = system_scope_error(config, platform_name, system_filter, target_name)
emu_profiles or {}, print(f" WARNING: {reason}")
target_cores, return None
platform_cores=plat_cores,
)
if system_filter:
filtered = select_systems(pack_systems, system_filter)
if not filtered:
hint = suggest_names(
system_filter,
sorted(pack_systems),
f"--platform {platform_name} --list-systems",
)
print(
f" WARNING: {platform_name} has no system matching "
f"{', '.join(system_filter)}. {hint}"
)
return None
pack_systems = filtered
if extras_only: if extras_only:
# The part of a split that carries the core extras no platform system # The part of a split that carries the core extras no platform system
# owns: the platform's own files and trees went to the other parts. # owns: the platform's own files and trees went to the other parts.
@@ -1882,11 +1867,6 @@ def generate_system_pack(
shutil.rmtree(scratch, ignore_errors=True) shutil.rmtree(scratch, ignore_errors=True)
def list_platforms(platforms_dir: str) -> list[str]:
"""List available platform names from registry."""
return list_registered_platforms(platforms_dir, include_archived=True)
def _system_display_name(system_id: str) -> str: def _system_display_name(system_id: str) -> str:
"""Convert system ID to display name for ZIP naming.""" """Convert system ID to display name for ZIP naming."""
s = system_id.lower().replace("_", "-") s = system_id.lower().replace("_", "-")
@@ -1935,7 +1915,6 @@ def generate_split_packs(
) -> list[str]: ) -> list[str]:
"""Generate split packs (one ZIP per system or manufacturer).""" """Generate split packs (one ZIP per system or manufacturer)."""
config = load_platform_config(platform_name, platforms_dir) config = load_platform_config(platform_name, platforms_dir)
platform_display = config.get("platform", platform_name)
# The split directory carries the same tags as the packs inside it, or two # The split directory carries the same tags as the packs inside it, or two
# differently narrowed runs write into one another. # differently narrowed runs write into one another.
split_tags = "".join( split_tags = "".join(
@@ -1951,13 +1930,19 @@ def generate_split_packs(
# The parts carry the version, so the directory does too: a rescrape that # The parts carry the version, so the directory does too: a rescrape that
# moves the version would otherwise leave the old parts beside the new # moves the version would otherwise leave the old parts beside the new
# under one SHA256SUMS.txt. # under one SHA256SUMS.txt.
version = config.get("version", config.get("dat_version", "")) # Named by the stem the full pack reads: assembled here by hand, a '/' in
ver_tag = f"_{version.replace(' ', '')}" if version else "" # a version nested the directory where the check never looked for it.
part_prefix = f"{platform_display.replace(' ', '_')}{ver_tag}{split_tags}" part_prefix = (
f"{_platform_pack_stem([platform_name], platform_name, platforms_dir)}{split_tags}"
)
split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split") split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split")
os.makedirs(split_dir, exist_ok=True) os.makedirs(split_dir, exist_ok=True)
systems = config.get("systems", {}) if emu_profiles is None:
emu_profiles = load_emulator_profiles(emulators_dir)
# The systems a core of the target runs: a part per declared system sent
# each one the target drops to a refusal naming it as unknown.
systems = platform_scope_systems(config, emu_profiles, target_cores)
if group_by == "manufacturer": if group_by == "manufacturer":
groups = _group_systems_by_manufacturer(systems, db, bios_dir) groups = _group_systems_by_manufacturer(systems, db, bios_dir)
@@ -1966,8 +1951,6 @@ def generate_split_packs(
# Pre-compute core extras once (expensive: scans 260+ emulator profiles) # Pre-compute core extras once (expensive: scans 260+ emulator profiles)
# then distribute per group based on emulator system overlap # then distribute per group based on emulator system overlap
if emu_profiles is None:
emu_profiles = load_emulator_profiles(emulators_dir)
base_dest = config.get("base_destination", "") base_dest = config.get("base_destination", "")
# Each part cuts the platform's own entries from the full profiles. The # Each part cuts the platform's own entries from the full profiles. The
# extras are collected once here, already without the cores' game data. # extras are collected once here, already without the cores' game data.
@@ -2373,6 +2356,46 @@ def _resolve_systems(args: argparse.Namespace, parser: argparse.ArgumentParser)
parser.error(f"--system {args.system}: {exc}") parser.error(f"--system {args.system}: {exc}")
def _platforms_with_systems(
args: argparse.Namespace,
platforms: list[str],
system_filter: list[str],
emu_profiles: dict,
target_cores_cache: dict[str, set[str] | None],
) -> list[str]:
"""The platforms a --system list leaves a system to pack, before any lock.
Under --all a platform without one is out of scope, as a platform
without the target is: `--all --system sony-playstation-2` built every
PS2 pack and exited 1 on MiSTer, which has no PS2. Under --platform the
request is refused here, with its reason.
"""
kept = []
for name in platforms:
config = load_platform_config(name, args.platforms_dir)
tc = target_cores_cache.get(name) if args.target else None
if platform_scope_systems(config, emu_profiles, tc, system_filter):
kept.append(name)
continue
if not args.all:
reason = system_scope_error(config, name, system_filter, args.target)
print(f"ERROR: {reason}", file=sys.stderr)
sys.exit(1)
declared = select_systems(config.get("systems", {}), system_filter)
reason = (
system_scope_error(config, name, system_filter, args.target)
if declared and args.target
else f"no system {', '.join(system_filter)}"
)
print(f"INFO: Skipping {name}: {reason}")
if not kept:
where = f" on target {args.target}" if args.target else ""
print(f"ERROR: no platform packed here runs {', '.join(system_filter)}{where}",
file=sys.stderr)
sys.exit(1)
return kept
def _refuse_for_verify_packs(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: def _refuse_for_verify_packs(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
"""What --verify-packs cannot be combined with, refused before any lock. """What --verify-packs cannot be combined with, refused before any lock.
@@ -2652,6 +2675,19 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
else: else:
raise ValueError("--verify-packs requires --platform or --all") raise ValueError("--verify-packs requires --platform or --all")
# The target is checked as the build checks it: a mistyped one, or one
# the platform does not publish, was dropped and the full pack verified
# under it, `all OK` for a pack no build had made.
target_cores_cache: dict[str, set[str] | None] = {}
if args.target:
try:
target_cores_cache, platforms = build_target_cores_cache(
platforms, args.target, args.platforms_dir, is_all=args.all
)
except (FileNotFoundError, ValueError) as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
all_ok = True all_ok = True
verify_regions = getattr(args, "regions", None) verify_regions = getattr(args, "regions", None)
verify_profiles = load_emulator_profiles(args.emulators_dir) verify_profiles = load_emulator_profiles(args.emulators_dir)
@@ -2689,7 +2725,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
else: else:
print(f" {platform_name}: SKIP (no pack in {args.output_dir})") print(f" {platform_name}: SKIP (no pack in {args.output_dir})")
continue continue
tc = _target_cores_for(platform_name, args.target, args.platforms_dir) tc = target_cores_cache.get(platform_name)
# A group pack and a pack built for the platform alone can both be # A group pack and a pack built for the platform alone can both be
# present, and each one is a pack of this platform. # present, and each one is a pack of this platform.
for pack_name in found: for pack_name in found:
@@ -3057,7 +3093,7 @@ def _refuse_unapplied_flags(args: argparse.Namespace, parser: argparse.ArgumentP
name for name, on in ( name for name, on in (
("--manifest-targets", args.manifest_targets), ("--manifest-targets", args.manifest_targets),
("--verify-packs", args.verify_packs and not args.all_variants), ("--verify-packs", args.verify_packs and not args.all_variants),
("--list", args.list), ("--list-platforms", args.list),
("--list-emulators", args.list_emulators), ("--list-emulators", args.list_emulators),
("--list-systems", args.list_systems), ("--list-systems", args.list_systems),
("--list-targets", args.list_targets), ("--list-targets", args.list_targets),
@@ -3083,7 +3119,7 @@ QUICK_MODE_FLAGS: dict[str, set[str]] = {
}, },
# One manifest per hardware target, from the target files themselves. # One manifest per hardware target, from the target files themselves.
"--manifest-targets": set(), "--manifest-targets": set(),
"--list": set(), "--list-platforms": set(),
"--list-emulators": set(), "--list-emulators": set(),
"--list-systems": {"--platform"}, "--list-systems": {"--platform"},
"--list-targets": {"--platform"}, "--list-targets": {"--platform"},
@@ -3112,7 +3148,7 @@ def _given_flags(args: argparse.Namespace) -> set[str]:
"--target": args.target, "--target": args.target,
"--region": args.region, "--region": args.region,
"--one-per-slot": args.one_per_slot, "--one-per-slot": args.one_per_slot,
"--list": args.list, "--list-platforms": args.list,
"--list-emulators": args.list_emulators, "--list-emulators": args.list_emulators,
"--list-systems": args.list_systems, "--list-systems": args.list_systems,
"--list-targets": args.list_targets, "--list-targets": args.list_targets,
@@ -3209,7 +3245,11 @@ def _main(held: contextlib.ExitStack) -> None:
action="store_true", action="store_true",
help="Pack the data directories as cached, still fetching release assets", help="Pack the data directories as cached, still fetching release assets",
) )
parser.add_argument("--list", action="store_true", help="List available platforms") # install.py's name. --list stays for the scripts that already type it.
parser.add_argument(
"--list-platforms", "--list", dest="list", action="store_true",
help="List available platforms",
)
parser.add_argument( parser.add_argument(
"--required-only", "--required-only",
action="store_true", action="store_true",
@@ -3311,14 +3351,7 @@ def _main(held: contextlib.ExitStack) -> None:
) )
return return
if args.list: if args.list:
# Every platform builds with --platform. --all builds the release print_platform_listing(args.platforms_dir)
# packs only, the others are served by the installer.
active = set(list_registered_platforms(args.platforms_dir))
released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True))
for p in list_platforms(args.platforms_dir):
marks = [m for m, on in (("archived", p not in active),
("installer only", p not in released)) if on]
print(f"{p} ({', '.join(marks)})" if marks else p)
return return
if args.list_emulators: if args.list_emulators:
list_emulator_profiles(args.emulators_dir) list_emulator_profiles(args.emulators_dir)
@@ -3464,6 +3497,28 @@ def _main(held: contextlib.ExitStack) -> None:
parser.error("Specify --platform or --all") parser.error("Specify --platform or --all")
return return
# A mistyped target is refused before the output lock and the data
# directory refresh: checked after them, `--target swich` waited on a
# pipeline's caches before saying the target does not exist.
target_cores_cache: dict[str, set[str] | None] = {}
if args.target:
try:
target_cores_cache, platforms = build_target_cores_cache(
platforms,
args.target,
args.platforms_dir,
is_all=args.all,
)
except (FileNotFoundError, ValueError) as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
emu_profiles = load_emulator_profiles(args.emulators_dir)
if system_filter:
platforms = _platforms_with_systems(
args, platforms, system_filter, emu_profiles, target_cores_cache
)
data_registry = load_data_dir_registry(args.platforms_dir) data_registry = load_data_dir_registry(args.platforms_dir)
# The output directory is claimed before the data directory refresh: a # The output directory is claimed before the data directory refresh: a
# refresh waits for every run that holds a cache, so a second build # refresh waits for every run that holds a cache, so a second build
@@ -3483,21 +3538,6 @@ def _main(held: contextlib.ExitStack) -> None:
if data_registry: if data_registry:
hold_data_dirs(held, data_registry) hold_data_dirs(held, data_registry)
emu_profiles = load_emulator_profiles(args.emulators_dir)
target_cores_cache: dict[str, set[str] | None] = {}
if args.target:
try:
target_cores_cache, platforms = build_target_cores_cache(
platforms,
args.target,
args.platforms_dir,
is_all=args.all,
)
except (FileNotFoundError, ValueError) as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
groups = group_identical_platforms( groups = group_identical_platforms(
platforms, args.platforms_dir, target_cores_cache if args.target else None platforms, args.platforms_dir, target_cores_cache if args.target else None
) )
@@ -4877,17 +4917,12 @@ def _target_cores_for(
) -> set[str] | None: ) -> set[str] | None:
"""Cores a target leaves available, or None when no target was asked for. """Cores a target leaves available, or None when no target was asked for.
None means no narrowing, which is what an unfiltered pack expects. Read Called on a platform the build already packed under the target, so the
the way --all builds: a platform without a target file was packed target is known to it: a platform without one is left out of the run.
unnarrowed under the target's name, and checking that pack must not
raise where building it did not.
""" """
if not target_name: if not target_name:
return None return None
try: return load_target_config(platform_name, target_name, platforms_dir)
return load_target_config(platform_name, target_name, platforms_dir)
except (FileNotFoundError, ValueError):
return None
def _expected_pack_names( def _expected_pack_names(
+14 -1
View File
@@ -244,6 +244,7 @@ class ExtraResolver:
offline: bool | None = True, offline: bool | None = True,
*, *,
profiles: dict, profiles: dict,
pending_assets: bool = False,
) -> None: ) -> None:
self.profiles = profiles self.profiles = profiles
self.db = db self.db = db
@@ -253,10 +254,13 @@ class ExtraResolver:
self.validation_index = validation_index self.validation_index = validation_index
self.validation_owners = validation_owners self.validation_owners = validation_owners
self.offline = offline self.offline = offline
# verify's replay: never downloads, and counts the release asset an
# online build fetches, as it does for a platform entry.
self.pending_assets = pending_assets
def resolve(self, fe: dict, dest: str) -> tuple[str | None, str, str]: def resolve(self, fe: dict, dest: str) -> tuple[str | None, str, str]:
"""(local copy or None, resolver status, the core's disagreement).""" """(local copy or None, resolver status, the core's disagreement)."""
from packresolve import resolve_file from packresolve import pending_release_asset, resolve_file, run_is_offline
from validation import core_size_refusal, extra_alternatives, validated_choice from validation import core_size_refusal, extra_alternatives, validated_choice
local_path, status = resolve_file( local_path, status = resolve_file(
@@ -264,6 +268,15 @@ class ExtraResolver:
dest_hint=fe.get("destination", ""), dest_hint=fe.get("destination", ""),
data_dir_registry=self.data_registry, offline=self.offline, data_dir_registry=self.data_registry, offline=self.offline,
) )
if (
self.pending_assets and status in ("not_found", "hash_mismatch")
and not run_is_offline()
):
# The asset the online builder fetches in resolve_file. Its record
# answers the declaration, and there are no bytes on disk to check.
asset = pending_release_asset(fe, self.db)
if asset:
return asset, "release_asset", ""
if status in ("not_found", "external", "user_provided") or not local_path: if status in ("not_found", "external", "user_provided") or not local_path:
return None, status, "" return None, status, ""
local_path, disagreement = validated_choice( local_path, disagreement = validated_choice(
+24 -9
View File
@@ -30,7 +30,7 @@ from packextras import platform_region_groups
from packpaths import _flat from packpaths import _flat
from nativemode import reads_file_contents from nativemode import reads_file_contents
import region as region_mod import region as region_mod
from packresolve import resolve_file from packresolve import resolve_declaration, resolve_file
from exporter.candidates import BIOS, EntryCategories from exporter.candidates import BIOS, EntryCategories
from gamedata import without_game_data from gamedata import without_game_data
from common import resolve_platform_cores from common import resolve_platform_cores
@@ -544,6 +544,21 @@ def _structural_errors(zf: zipfile.ZipFile, zip_set: set) -> list[str]:
return errors return errors
def _shipped_copy(
entry: dict, db: dict, data_registry: dict | None, mode: str, hash_rule: str,
) -> str | None:
"""The file the builder packs for a declaration, or None.
The verdict verify applies before it lets a declaration hold its path.
"""
local, status = resolve_declaration(
entry, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True,
)
if local and baseline_verdict(entry, local, status, mode, hash_rule)[1] is None:
return local
return None
def _builder_extras( def _builder_extras(
config: dict, config: dict,
emulators_dir: str, emulators_dir: str,
@@ -563,15 +578,17 @@ def _builder_extras(
"""{full destination: local copy} of each core extra the builder writes. """{full destination: local copy} of each core extra the builder writes.
The platform's paths come first, as in the builder: a destination the The platform's paths come first, as in the builder: a destination the
pack holds whose declarations resolve to a file, or that the platform pack holds whose declarations resolve to a file it ships, or that the
keeps for a user_provided or external entry. One whose declarations platform keeps for a user_provided or external entry. One whose
resolve to nothing is left free, since the builder then places a core's declarations resolve to nothing, or to a file the frontend's hash
file there. refuses, is left free, since the builder then places a core's file
there: taken by the refused file, that core file was never compared.
""" """
from packextras import ExtraResolver, hidden_extras from packextras import ExtraResolver, hidden_extras
from packresolve import resolve_declaration
from validation import _build_validation_index, destination_owners from validation import _build_validation_index, destination_owners
mode = config.get("verification_mode", "existence")
hash_rule = config.get("hash_rule", "")
shipped: dict[str, str | None] = {} shipped: dict[str, str | None] = {}
for expected, declarations in baseline_groups.items(): for expected, declarations in baseline_groups.items():
if expected not in zip_set and expected.lower() not in zip_lower: if expected not in zip_set and expected.lower() not in zip_lower:
@@ -587,9 +604,7 @@ def _builder_extras(
local = next( local = next(
( (
found for fe in declarations found for fe in declarations
if (found := resolve_declaration( if (found := _shipped_copy(fe, db, data_registry, mode, hash_rule))
fe, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True,
)[0])
), ),
None, None,
) )
+23 -5
View File
@@ -100,6 +100,20 @@ class Conflict:
def emulators(self) -> list[str]: def emulators(self) -> list[str]:
return sorted({c.emulator for c in self.profile_claims if c.emulator}) return sorted({c.emulator for c in self.profile_claims if c.emulator})
def files_read(self) -> list[tuple[str | None, list[str], str]]:
"""Each file the profiles want here, with the emulators reading it
and the proof of the first claim on it.
One file stood for every claim: Recalbox's neocdz.zip line gave
geolith the archive fbneo reads, not the one geolith pins.
"""
files: dict[str | None, tuple[list[str], str]] = {}
for claim in self.profile_claims:
emulators, status = files.setdefault(claim.local_path, ([], claim.status))
if claim.emulator and claim.emulator not in emulators:
emulators.append(claim.emulator)
return [(path, sorted(emus), status) for path, (emus, status) in files.items()]
def _normalize(destination: str, fold: bool = True) -> str: def _normalize(destination: str, fold: bool = True) -> str:
"""Comparable form of a destination, since layers write it differently. """Comparable form of a destination, since layers write it differently.
@@ -661,11 +675,13 @@ def format_decision(decision: Decision) -> str:
f"{conflict.destination}: serve {decision.winner.local_path}, " f"{conflict.destination}: serve {decision.winner.local_path}, "
"the pack answers to the emulator" "the pack answers to the emulator"
) )
expected = ", ".join(conflict.emulators) or "the profile" rejected = ", ".join(
f"{path}, the file {', '.join(emulators) or 'the profile'} reads"
for path, emulators, _status in conflict.files_read()
)
return ( return (
f"{conflict.destination}: keep {decision.winner.local_path}, the frontend " f"{conflict.destination}: keep {decision.winner.local_path}, the frontend "
f"verifies content and would reject " f"verifies content and would reject {rejected}"
f"{conflict.profile_claims[0].local_path}, the file {expected} reads"
) )
@@ -996,8 +1012,10 @@ def main() -> int:
"ships": c.platform_claim.local_path, "ships": c.platform_claim.local_path,
"ships_evidence": c.platform_claim.status, "ships_evidence": c.platform_claim.status,
"emulators": c.emulators, "emulators": c.emulators,
"expected": c.profile_claims[0].local_path, "expected": [
"expected_evidence": c.profile_claims[0].status, {"file": path, "evidence": status, "emulators": emulators}
for path, emulators, status in c.files_read()
],
} }
for c in conflicts for c in conflicts
] ]
+78 -33
View File
@@ -29,6 +29,7 @@ import contextlib
import json import json
import os import os
import sys import sys
from collections import Counter
from collections.abc import Mapping from collections.abc import Mapping
from pathlib import Path from pathlib import Path
@@ -54,7 +55,6 @@ from common import (
compute_hashes, compute_hashes,
expand_directory_entries, expand_directory_entries,
platform_declarations, platform_declarations,
filter_systems_by_target,
group_identical_platforms, group_identical_platforms,
list_emulator_profiles, list_emulator_profiles,
list_system_ids, list_system_ids,
@@ -63,7 +63,10 @@ from common import (
load_emulator_profiles, load_emulator_profiles,
load_platform_config, load_platform_config,
resolve_system_arg, resolve_system_arg,
print_platform_listing,
select_systems, select_systems,
platform_scope_systems,
system_scope_error,
md5sum, md5sum,
require_yaml, require_yaml,
resolve_local_file, resolve_local_file,
@@ -1217,16 +1220,7 @@ def verify_platform(
validation_index = _build_validation_index(platform_profiles) validation_index = _build_validation_index(platform_profiles)
validation_owners = destination_owners(platform_profiles) validation_owners = destination_owners(platform_profiles)
if not target_cores: verify_systems = platform_scope_systems(config, profiles, target_cores, system_filter)
plat_cores = None
verify_systems = filter_systems_by_target(
config.get("systems", {}),
profiles,
target_cores,
platform_cores=plat_cores,
)
if system_filter:
verify_systems = select_systems(verify_systems, system_filter)
details = [] details = []
file_status: dict[str, str] = {} file_status: dict[str, str] = {}
@@ -1315,8 +1309,12 @@ def verify_platform(
"release_asset": local_path, "release_asset": local_path,
"reason": "release asset, fetched when the pack is built", "reason": "release asset, fetched when the pack is built",
} }
# Written at its path like any held file: keyed to None, it
# stayed out of pack_destinations and step 5 read the pack's
# copy as a file only the pack holds.
packed_order.append(( packed_order.append((
sys_id, f"{base_dest}/{destination}" if base_dest else destination, None, sys_id, f"{base_dest}/{destination}" if base_dest else destination,
local_path,
)) ))
shipped_keys.add(_dest_key(file_entry, sys_id)) shipped_keys.add(_dest_key(file_entry, sys_id))
local_path = None local_path = None
@@ -1465,7 +1463,7 @@ def verify_platform(
region_drops, report_for_builder, region_drops, report_for_builder,
ExtraResolver( ExtraResolver(
db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index, db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index,
validation_owners, profiles=profiles, validation_owners, profiles=profiles, pending_assets=True,
), ),
placed_kind, placed_kind,
system_filter, system_filter,
@@ -1746,13 +1744,20 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) ->
print(f" {line}") print(f" {line}")
def _print_undeclared_entry(u: dict, prefix: str, verbose: bool) -> None: def _print_undeclared_entry(
"""Print a single undeclared file entry with its validation checks.""" u: dict, prefix: str, verbose: bool, repeated: frozenset[str] = frozenset()
) -> None:
"""Print a single undeclared file entry with its validation checks.
A name in ``repeated`` is read at more than one path, and the line names
the path: two identical lines told no one there were two slots to fill.
"""
name = u["path"] if u["name"] in repeated and u.get("path") else u["name"]
arc_count = u.get("archive_file_count") arc_count = u.get("archive_file_count")
if arc_count: if arc_count:
name_label = f"{u['name']} ({arc_count} file{'s' if arc_count != 1 else ''})" name_label = f"{name} ({arc_count} file{'s' if arc_count != 1 else ''})"
else: else:
name_label = u["name"] name_label = name
print(f" {prefix}: {u['emulator']} needs {name_label}") print(f" {prefix}: {u['emulator']} needs {name_label}")
checks = u.get("checks", []) checks = u.get("checks", [])
if checks: if checks:
@@ -1800,6 +1805,25 @@ def _core_required_missing(bios_files: list[dict]) -> list[dict]:
] ]
def _repeated_names(entries: list[dict]) -> frozenset[str]:
"""Names more than one of ``entries`` carries."""
counts = Counter(u["name"] for u in entries)
return frozenset(name for name, count in counts.items() if count > 1)
def _omission_reasons(entries: list[dict]) -> str:
"""Each reason a held file was left out, with its count when several.
Printing the first entry's reason for the whole count named one cause
for 412 RomM files when 20 carried it.
"""
counts = Counter(u.get("omitted", "not placed") for u in entries)
if len(counts) == 1:
return next(iter(counts))
ranked = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
return ", ".join(f"{count} {reason}" for reason, count in ranked)
def _print_undeclared_section(result: dict, verbose: bool) -> None: def _print_undeclared_section(result: dict, verbose: bool) -> None:
"""Print cross-reference section for undeclared files used by cores.""" """Print cross-reference section for undeclared files used by cores."""
undeclared = result.get("undeclared_files", []) undeclared = result.get("undeclared_files", [])
@@ -1839,19 +1863,22 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, " f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, "
f"{core_missing_opt} optional missing{listed}" f"{core_missing_opt} optional missing{listed}"
) )
# The missing files follow the line that counts them: printed after the
# held-but-not-packed line, they read as files installed with their emulator.
repeated = _repeated_names(req_not_in_repo + req_hle_not_in_repo + opt_not_in_repo)
for u in req_not_in_repo:
_print_undeclared_entry(u, "MISSING (required)", verbose, repeated)
for u in req_hle_not_in_repo:
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose, repeated)
if verbose:
for u in opt_not_in_repo:
_print_undeclared_entry(u, "MISSING (optional)", verbose, repeated)
if bundled: if bundled:
names = ", ".join(sorted({u["name"] for u in bundled})) names = ", ".join(sorted({u["name"] for u in bundled}))
print(f" Bundled with their emulator, not collected: {names}") print(f" Bundled with their emulator, not collected: {names}")
if unplaced: if unplaced:
print(f" Core files held but not packed: {len(unplaced)} ({unplaced[0]['omitted']})") print(f" Core files held but not packed: {len(unplaced)} "
f"({_omission_reasons(unplaced)})")
for u in req_not_in_repo:
_print_undeclared_entry(u, "MISSING (required)", verbose)
for u in req_hle_not_in_repo:
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose)
if verbose:
for u in opt_not_in_repo:
_print_undeclared_entry(u, "MISSING (optional)", verbose)
if game_data: if game_data:
gd_missing = [u for u in game_data if not u["in_repo"]] gd_missing = [u for u in game_data if not u["in_repo"]]
@@ -1864,12 +1891,13 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
listed = "" if verbose or not gd_missing else " (--verbose lists them)" listed = "" if verbose or not gd_missing else " (--verbose lists them)"
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}") print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}")
if verbose: if verbose:
repeated = _repeated_names(gd_missing)
for u in gd_missing: for u in gd_missing:
_print_undeclared_entry(u, "MISSING (game data)", verbose) _print_undeclared_entry(u, "MISSING (game data)", verbose, repeated)
if gd_unplaced: if gd_unplaced:
print( print(
f" Game data held but not packed: {len(gd_unplaced)} " f" Game data held but not packed: {len(gd_unplaced)} "
f"({gd_unplaced[0].get('omitted', 'not placed')})" f"({_omission_reasons(gd_unplaced)})"
) )
@@ -2597,6 +2625,7 @@ def _refuse_listing_narrowings(
""" """
listing = next( listing = next(
(flag for flag, on in ( (flag for flag, on in (
("--list-platforms", args.list_platforms),
("--list-emulators", args.list_emulators), ("--list-emulators", args.list_emulators),
("--list-systems", args.list_systems), ("--list-systems", args.list_systems),
("--list-targets", args.list_targets), ("--list-targets", args.list_targets),
@@ -2605,7 +2634,7 @@ def _refuse_listing_narrowings(
) )
if not listing: if not listing:
return return
reads_platform = listing != "--list-emulators" reads_platform = listing not in ("--list-emulators", "--list-platforms")
for flag, on in ( for flag, on in (
("--platform", args.platform and not reads_platform), ("--platform", args.platform and not reads_platform),
("--all", args.all), ("--all", args.all),
@@ -2655,7 +2684,9 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
def _run_listing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: def _run_listing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
if args.list_emulators: if args.list_platforms:
print_platform_listing(args.platforms_dir)
elif args.list_emulators:
list_emulator_profiles(args.emulators_dir) list_emulator_profiles(args.emulators_dir)
elif args.list_systems and args.platform: elif args.list_systems and args.platform:
list_platform_system_ids(args.platform, args.platforms_dir) list_platform_system_ids(args.platform, args.platforms_dir)
@@ -2694,6 +2725,9 @@ def _main(held: contextlib.ExitStack) -> None:
parser.add_argument( parser.add_argument(
"--list-systems", action="store_true", help="List available systems" "--list-systems", action="store_true", help="List available systems"
) )
parser.add_argument(
"--list-platforms", action="store_true", help="List available platforms"
)
parser.add_argument( parser.add_argument(
"--include-archived", action="store_true", "--include-archived", action="store_true",
help="With --all, also verify the archived platforms (RetroPie)", help="With --all, also verify the archived platforms (RetroPie)",
@@ -2767,7 +2801,7 @@ def _main(held: contextlib.ExitStack) -> None:
parser.error(str(exc)) parser.error(str(exc))
_refuse_listing_narrowings(args, parser) _refuse_listing_narrowings(args, parser)
if args.list_emulators or args.list_systems or args.list_targets: if args.list_platforms or args.list_emulators or args.list_systems or args.list_targets:
_run_listing(args, parser) _run_listing(args, parser)
return return
@@ -2853,6 +2887,18 @@ def _main(held: contextlib.ExitStack) -> None:
except (FileNotFoundError, ValueError) as e: except (FileNotFoundError, ValueError) as e:
print(f"ERROR: {e}", file=sys.stderr) print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1) sys.exit(1)
# A system the target leaves no core for was reported 0/0 with exit 0,
# where generate_pack refuses the same request.
if platform_systems:
config = load_platform_config(args.platform, args.platforms_dir)
if not platform_scope_systems(
config, emu_profiles, target_cores_cache.get(args.platform), platform_systems
):
reason = system_scope_error(
config, args.platform, platform_systems, args.target
)
print(f"ERROR: {reason}", file=sys.stderr)
sys.exit(1)
# Group identical platforms (same function as generate_pack) # Group identical platforms (same function as generate_pack)
groups = group_identical_platforms( groups = group_identical_platforms(
@@ -2961,8 +3007,7 @@ def _overall_summary(group_results: list[tuple[dict, list[str]]]) -> str:
platforms = sum(len(group) for _, group in group_results) platforms = sum(len(group) for _, group in group_results)
def where(counts: dict[str, int]) -> str: def where(counts: dict[str, int]) -> str:
named = [f"{name} {count}" if count > 1 else name named = [f"{name} {count}" for name, count in counts.items() if count]
for name, count in counts.items() if count]
return f" ({', '.join(named)})" if named else "" return f" ({', '.join(named)})" if named else ""
parts = [] parts = []
+1 -1
View File
@@ -4830,7 +4830,7 @@ class VerifyAllSummaryNamesWhatIsWrong(unittest.TestCase):
summary = verify._overall_summary( summary = verify._overall_summary(
[(clean, ["RetroArch"]), (warned, ["RetroDECK"]), (core_miss, ["ROCKNIX"])] [(clean, ["RetroArch"]), (warned, ["RetroDECK"]), (core_miss, ["ROCKNIX"])]
) )
self.assertIn("1 warning (RetroDECK)", summary) self.assertIn("1 warning (RetroDECK 1)", summary)
self.assertIn("ROCKNIX: n64.psh", summary) self.assertIn("ROCKNIX: n64.psh", summary)
self.assertIn("0 critical,", summary) self.assertIn("0 critical,", summary)
+53
View File
@@ -87,6 +87,59 @@ class CoreFilesAsTheBuilderPlacesThem(unittest.TestCase):
) )
class ARefusedPinLeavesItsPathToTheCore(unittest.TestCase):
"""EmuELEC pins naomi.zip by an md5 the collection does not hold: the
builder refuses the held copy and ships the core's file at that path.
The check marked the path taken by the refused platform file, so the
core file the pack carries there was never compared."""
def setUp(self):
tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
self.addCleanup(tmp.cleanup)
previous = os.getcwd()
os.chdir(tmp.name)
self.addCleanup(os.chdir, previous)
files: dict = {}
by_name: dict = {}
_held(files, by_name, "bios/dc/naomi.zip", b"the core's naomi")
self.db = {"files": files, "indexes": {
"by_name": by_name, "by_md5": {e["md5"]: s for s, e in files.items()},
"by_path_suffix": {}, "by_crc32": {}, "by_sha256": {}}}
Path("platforms").mkdir()
config = {"platform": "P", "verification_mode": "md5", "cores": ["flycast"],
"systems": {"dc": {"files": [{
"name": "naomi.zip", "destination": "dc/naomi.zip",
"md5": hashlib.md5(b"a bundle nobody holds").hexdigest(),
}]}}}
Path("platforms/p.yml").write_text(yaml.safe_dump(config), encoding="utf-8")
Path("emulators").mkdir()
self.profiles = {"flycast": {
"emulator": "flycast", "type": "libretro", "systems": ["dc"], "files": [
{"name": "naomi.zip", "path": "dc/naomi.zip"}]}}
def _check(self, member: bytes) -> tuple:
with zipfile.ZipFile("pack.zip", "w") as zf:
zf.writestr("dc/naomi.zip", member)
return packverify.verify_pack_against_platform(
"pack.zip", "p", "platforms", self.db, emulators_dir="emulators",
emu_profiles=self.profiles,
)
def test_the_core_file_at_the_refused_path_is_compared(self):
_ok, _c, _p, errors, *_mid, core_checked, _cp, _ex, _ab = self._check(b"other")
self.assertEqual(core_checked, 1)
self.assertTrue(
any(e.startswith("core file differs: dc/naomi.zip") for e in errors), errors
)
def test_the_right_core_file_passes(self):
_ok, _c, _p, errors, *_mid, core_checked, core_present, _ex, _ab = (
self._check(b"the core's naomi")
)
self.assertEqual((core_checked, core_present), (1, 1))
self.assertFalse([e for e in errors if e.startswith("core")], errors)
class NarrowedPacksAgainstTheirOwnList(unittest.TestCase): class NarrowedPacksAgainstTheirOwnList(unittest.TestCase):
"""--verify-packs after --all-variants said "hashes checked during the """--verify-packs after --all-variants said "hashes checked during the
build" for the five narrowed packs and compared none of them: a narrowed build" for the five narrowed packs and compared none of them: a narrowed
+18
View File
@@ -420,6 +420,24 @@ class TestArbitration(unittest.TestCase):
served = slots.format_decision(slots.arbitrate(self._conflict(), "existence")) served = slots.format_decision(slots.arbitrate(self._conflict(), "existence"))
self.assertIn("both are satisfied", served) self.assertIn("both are satisfied", served)
def test_each_emulator_is_given_the_file_it_reads(self):
"""Recalbox's neocdz.zip line said geolith reads the archive fbneo
reads, while geolith pins another one."""
conflict = self._conflict()
conflict.profile_claims = [
slots.Claim("profile", "neocdz.zip", "neocdz.zip", emulator="fbneo",
local_path="a/neocdz.zip.ba20a7be", status="members_held"),
slots.Claim("profile", "neocdz.zip", "neocdz.zip", emulator="geolith",
local_path="b/neocdz.zip.712816a3", status="md5_exact"),
]
kept = slots.format_decision(slots.arbitrate(conflict, "md5"))
self.assertIn("a/neocdz.zip.ba20a7be, the file fbneo reads", kept)
self.assertIn("b/neocdz.zip.712816a3, the file geolith reads", kept)
self.assertEqual(conflict.files_read(), [
("a/neocdz.zip.ba20a7be", ["fbneo"], "members_held"),
("b/neocdz.zip.712816a3", ["geolith"], "md5_exact"),
])
class TheStrongestProofTakesAnUndeclaredPath(unittest.TestCase): class TheStrongestProofTakesAnUndeclaredPath(unittest.TestCase):
"""No platform declares RetroDECK's hiscore.dat. fbalpha2012 asks for it """No platform declares RetroDECK's hiscore.dat. fbalpha2012 asks for it
+169 -2
View File
@@ -4,11 +4,13 @@
under the target's name (Recalbox, Lakka, BizHawk, MiSTer, ROCKNIX). The under the target's name (Recalbox, Lakka, BizHawk, MiSTer, ROCKNIX). The
conformance check read the same target without the --all tolerance, raised conformance check read the same target without the --all tolerance, raised
FileNotFoundError on the first of those packs and stopped the run in a FileNotFoundError on the first of those packs and stopped the run in a
traceback after every pack had been written. traceback after every pack had been written. Such a platform is now left
out of a targeted --all run, and the check refuses it as the build does.
""" """
from __future__ import annotations from __future__ import annotations
import os
import sys import sys
import contextlib import contextlib
import io import io
@@ -31,7 +33,10 @@ class CheckReadsTheTargetLikeTheBuild(unittest.TestCase):
contextlib.redirect_stdout(io.StringIO()): contextlib.redirect_stdout(io.StringIO()):
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
build_target_cores_cache(["recalbox"], "x86_64", platforms, is_all=True) build_target_cores_cache(["recalbox"], "x86_64", platforms, is_all=True)
self.assertIsNone(_target_cores_for("recalbox", "x86_64", platforms)) # The build never packs such a platform under the target, so the
# check refuses it too instead of reading the full pack.
with self.assertRaises(FileNotFoundError):
_target_cores_for("recalbox", "x86_64", platforms)
def test_a_platform_with_a_target_file_is_narrowed(self): def test_a_platform_with_a_target_file_is_narrowed(self):
cores = _target_cores_for("retroarch", "switch", str(REPO_ROOT / "platforms")) cores = _target_cores_for("retroarch", "switch", str(REPO_ROOT / "platforms"))
@@ -54,5 +59,167 @@ class ANameClaimsOnlyANarrowingThatHappened(unittest.TestCase):
) )
class AMistypedTargetIsRefusedFirst(unittest.TestCase):
"""`--platform retroarch --target swich` took the dist lock and waited on
the data directory refresh, 22 s here and hours behind a pipeline, before
saying the target does not exist."""
def test_no_lock_and_no_refresh_before_the_refusal(self):
from unittest import mock
import generate_pack
def touched(*_a, **_kw):
raise AssertionError("lock or refresh taken before the target check")
argv = ["generate_pack.py", "--platform", "retroarch", "--target", "swich",
"--output-dir", str(REPO_ROOT / "tmp" / "never-written")]
err = io.StringIO()
with mock.patch.object(sys, "argv", argv), \
mock.patch.object(generate_pack, "_pack_output_lock", touched), \
mock.patch.object(generate_pack, "hold_data_dirs", touched), \
contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(err), \
self.assertRaises(SystemExit) as ctx:
generate_pack.main()
self.assertEqual(ctx.exception.code, 1)
self.assertIn("Unknown target 'swich'", err.getvalue())
class ASystemTheTargetDropsIsNamedAsSuch(unittest.TestCase):
"""`--platform retroarch --system sony-playstation-2 --target switch`:
verify printed 0/0 with exit 0 under --strict, and generate_pack said
RetroArch has no such system and suggested sony-playstation."""
PLATFORMS = str(REPO_ROOT / "platforms")
def test_both_tools_read_one_scope(self):
from common import (
load_emulator_profiles, load_platform_config, platform_scope_systems,
)
config = load_platform_config("retroarch", self.PLATFORMS)
profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
switch = _target_cores_for("retroarch", "switch", self.PLATFORMS)
wanted = ["sony-playstation-2"]
self.assertEqual(platform_scope_systems(config, profiles, switch, wanted), {})
self.assertEqual(list(platform_scope_systems(config, profiles, None, wanted)), wanted)
def test_the_message_names_the_target(self):
from common import system_scope_error
config = {"systems": {"sony-playstation-2": {}, "sony-playstation": {}}}
self.assertEqual(
system_scope_error(config, "retroarch", ["sony-playstation-2"], "switch"),
"no core retroarch runs on target switch emulates sony-playstation-2",
)
undeclared = system_scope_error(config, "retroarch", ["nec-pc-98"], "switch")
self.assertTrue(undeclared.startswith("retroarch has no system matching nec-pc-98."))
def test_verify_refuses_it(self):
import subprocess
proc = subprocess.run(
[sys.executable, "scripts/verify.py", "--platform", "retroarch",
"--system", "sony-playstation-2", "--target", "switch", "--strict"],
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=600, check=False,
)
self.assertEqual(proc.returncode, 1, proc.stdout[-300:])
self.assertIn("no core retroarch runs on target switch emulates sony-playstation-2",
proc.stderr)
class ASplitBuildsOnlyTheTargetsSystems(unittest.TestCase):
"""`--split --target switch` asked for a part per declared system, and
each one the target drops ended on 'retroarch has no system matching'."""
def test_no_part_is_asked_for_a_dropped_system(self):
from unittest import mock
import generate_pack
from common import load_emulator_profiles
platforms = str(REPO_ROOT / "platforms")
profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
asked: list[list[str]] = []
def record(*_a, **kw):
if kw.get("system_filter"):
asked.append(kw["system_filter"])
return None
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \
mock.patch.object(generate_pack, "generate_pack", record), \
mock.patch.object(generate_pack, "_collect_emulator_extras",
return_value=[]):
generate_pack.generate_split_packs(
"retroarch", platforms, {"files": {}}, "bios", out,
emu_profiles=profiles,
target_cores=_target_cores_for("retroarch", "switch", platforms),
target_name="nintendo-switch",
)
systems = {sid for ids in asked for sid in ids}
self.assertTrue(systems)
self.assertNotIn("sony-playstation-2", systems)
self.assertIn("sony-playstation", systems)
class APlatformWithoutTheSystemIsOutOfScope(unittest.TestCase):
"""`--all --system sony-playstation-2` built the five PS2 packs and
exited 1 on 'no pack for misterfpga', which has no PS2."""
def _scope(self, **args):
import argparse
import generate_pack
from common import load_emulator_profiles
values = {"platforms_dir": str(REPO_ROOT / "platforms"), "target": None,
"all": True}
values.update(args)
out = io.StringIO()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
kept = generate_pack._platforms_with_systems(
argparse.Namespace(**values), ["retroarch", "misterfpga"],
["sony-playstation-2"],
load_emulator_profiles(str(REPO_ROOT / "emulators")), {},
)
return kept, out.getvalue()
def test_all_skips_it(self):
kept, out = self._scope()
self.assertEqual(kept, ["retroarch"])
self.assertIn("INFO: Skipping misterfpga: no system sony-playstation-2", out)
def test_a_named_platform_is_refused(self):
with self.assertRaises(SystemExit) as ctx:
self._scope(all=False)
self.assertEqual(ctx.exception.code, 1)
class TheSplitDirectoryReadsThePackStem(unittest.TestCase):
"""The split prefix was assembled by hand: a version 'release/1.22' made
'RetroArch_release/1.22_Split', a nested directory the check never
found, beside a full pack named 'RetroArch_release-1.22_BIOS_Pack.zip'."""
def test_a_reserved_character_is_made_safe_as_in_the_full_pack(self):
from unittest import mock
import generate_pack
config = {"platform": "Foo: Bar", "version": "release/1.22", "systems": {}}
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \
mock.patch.object(generate_pack, "load_platform_config",
return_value=config), \
mock.patch.object(generate_pack, "_collect_emulator_extras",
return_value=[]):
generate_pack.generate_split_packs(
"foo", "platforms", {"files": {}}, "bios", out, emu_profiles={},
)
stem = generate_pack._platform_pack_stem(["foo"], "foo", "platforms")
self.assertEqual(os.listdir(out), [f"{stem}_Split"])
self.assertEqual(stem, "Foo-_Bar_release-1.22")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+116 -1
View File
@@ -132,7 +132,7 @@ class TheVerifySummaryCountsDiscrepancies(unittest.TestCase):
({"severity_counts": {}, "details": details}, ["RetroArch"]), ({"severity_counts": {}, "details": details}, ["RetroArch"]),
({"severity_counts": {}, "details": []}, ["Batocera"]), ({"severity_counts": {}, "details": []}, ["Batocera"]),
]) ])
self.assertIn("1 discrepancy (RetroArch)", summary) self.assertIn("1 discrepancy (RetroArch 1)", summary)
def test_a_shared_destination_names_the_path_and_both_sides(self): def test_a_shared_destination_names_the_path_and_both_sides(self):
result = {"name": "disk.rom", "status": verify.Status.UNTESTED} result = {"name": "disk.rom", "status": verify.Status.UNTESTED}
@@ -145,6 +145,121 @@ class TheVerifySummaryCountsDiscrepancies(unittest.TestCase):
self.assertIn(part, message) self.assertIn(part, message)
class EveryPlatformInTheSummaryCarriesItsCount(unittest.TestCase):
"""'17 discrepancies (Lakka 5, RetroArch 5, RetroBat 2, RetroDECK 4,
ROCKNIX)': the one platform with a single entry had no number, and the
named counts did not add up to the total."""
def test_a_count_of_one_is_written(self):
one = {"severity_counts": {verify.Severity.WARNING: 1}, "details": []}
two = {"severity_counts": {verify.Severity.WARNING: 2}, "details": []}
summary = verify._overall_summary([(two, ["Lakka"]), (one, ["ROCKNIX"])])
self.assertIn("3 warning (Lakka 2, ROCKNIX 1)", summary)
class HeldButNotPackedNamesEachReason(unittest.TestCase):
"""'Core files held but not packed: 16 (a packed file occupies its
directory)' on Lakka, where 14 of the 16 are installed with their
emulator: the first entry's reason stood for the whole count."""
@staticmethod
def _entry(name: str, reason: str, category: str = "bios") -> dict:
return {"name": name, "emulator": "x", "required": True, "in_repo": True,
"in_pack": False, "omitted": reason, "category": category}
def _printed(self, entries: list[dict]) -> str:
out = io.StringIO()
with contextlib.redirect_stdout(out):
verify._print_undeclared_section({"undeclared_files": entries}, False)
return out.getvalue()
def test_each_reason_carries_its_count(self):
entries = [self._entry("a.rom", "a packed file occupies its directory")]
entries += [self._entry(f"{i}.rom", "installed with its emulator") for i in range(3)]
entries += [self._entry(f"g{i}", "installed with its emulator", "game_data")
for i in range(2)]
entries.append(self._entry("g9", "no platform slug for its system", "game_data"))
text = self._printed(entries)
self.assertIn(
"Core files held but not packed: 4 (3 installed with its emulator, "
"1 a packed file occupies its directory)", text)
self.assertIn(
"Game data held but not packed: 3 (2 installed with its emulator, "
"1 no platform slug for its system)", text)
def test_a_missing_file_follows_the_line_that_counts_it(self):
"""Indented under 'held but not packed: 102 (installed with its
emulator)', Batocera's one missing sfiii2a.zip read as one of them."""
missing = {"name": "sfiii2a.zip", "emulator": "FinalBurn Neo", "required": True,
"in_repo": False, "category": "bios"}
lines = self._printed(
[missing, self._entry("a.rom", "installed with its emulator")]
).splitlines()
self.assertTrue(lines[0].startswith(" Core files: "), lines)
self.assertIn("MISSING (required): FinalBurn Neo needs sfiii2a.zip", lines[1])
self.assertTrue(lines[2].startswith(" Core files held but not packed: "), lines)
def test_one_name_read_at_two_paths_prints_both_paths(self):
"""MyMinUI printed 'FinalBurn Neo needs sfiii2a.zip' twice, word for
word, for Bios/FBN/ and Bios/NG/."""
base = {"name": "sfiii2a.zip", "emulator": "FinalBurn Neo", "required": True,
"in_repo": False, "category": "bios"}
lone = dict(base, name="other.zip", path="Bios/FBN/other.zip")
text = self._printed([dict(base, path="Bios/FBN/sfiii2a.zip"),
dict(base, path="Bios/NG/sfiii2a.zip"), lone])
self.assertIn("FinalBurn Neo needs Bios/FBN/sfiii2a.zip", text)
self.assertIn("FinalBurn Neo needs Bios/NG/sfiii2a.zip", text)
self.assertIn("FinalBurn Neo needs other.zip\n", text)
def test_a_single_reason_reads_as_before(self):
text = self._printed([self._entry("a.rom", "installed with its emulator")] * 2)
self.assertIn("Core files held but not packed: 2 (installed with its emulator)", text)
class APlatformWithoutTargetsSaysSo(unittest.TestCase):
"""`--platform misterfpga --target switch` printed 'No target config for
platform 'misterfpga': platforms/targets/misterfpga.yml', which reads as
a file missing from the repository."""
def test_the_message_names_the_reason_and_the_platforms_with_targets(self):
with tempfile.TemporaryDirectory() as tmp:
(Path(tmp) / "targets").mkdir()
for name in ("retroarch", "batocera", "_overrides"):
(Path(tmp) / "targets" / f"{name}.yml").write_text("targets: {}\n")
with self.assertRaises(FileNotFoundError) as caught:
common.load_target_config("misterfpga", "switch", tmp)
self.assertEqual(
str(caught.exception),
"misterfpga has no hardware targets: drop --target "
"(platforms with targets: batocera, retroarch)",
)
class OneNameListsThePlatforms(unittest.TestCase):
"""install.py --list-platforms, generate_pack.py --list, and no listing
at all in verify.py: three scripts, three answers to one question."""
def _run(self, script: str, flag: str) -> subprocess.CompletedProcess:
return subprocess.run(
[sys.executable, f"scripts/{script}", flag], capture_output=True, text=True,
cwd=str(REPO_ROOT), timeout=120, check=False,
)
def test_both_tools_take_it_and_agree(self):
verify_out = self._run("verify.py", "--list-platforms")
pack_out = self._run("generate_pack.py", "--list-platforms")
self.assertEqual(verify_out.returncode, 0, verify_out.stderr)
self.assertEqual(pack_out.returncode, 0, pack_out.stderr)
self.assertEqual(verify_out.stdout, pack_out.stdout)
self.assertIn("retroarch", verify_out.stdout.split())
self.assertIn("--list-platforms", (REPO_ROOT / "install.py").read_text())
def test_the_installer_name_is_the_one_documented(self):
tools = (REPO_ROOT / "wiki" / "tools.md").read_text()
self.assertIn("generate_pack.py --list-platforms", tools)
self.assertIn("verify.py --list-platforms", tools)
class OneSpellingPerOption(unittest.TestCase): class OneSpellingPerOption(unittest.TestCase):
"""verify.py -p batocera worked and slots.py -p batocera was refused.""" """verify.py -p batocera worked and slots.py -p batocera was refused."""
+18
View File
@@ -116,6 +116,24 @@ class VerifyPackChoice(unittest.TestCase):
dict(zip((name for _p, name in checked), self.variants)), names dict(zip((name for _p, name in checked), self.variants)), names
) )
def test_a_target_the_platform_lacks_is_refused(self):
"""`--platform bizhawk --target rpi4typo --verify-packs` verified the
full pack and said `all OK`, where the build refused the target."""
import contextlib
import io
bizhawk = self.gp._platform_pack_stem(["bizhawk"], "bizhawk", "platforms")
self._touch(f"{bizhawk}_BIOS_Pack.zip")
self._touch(f"{self.stem}_BIOS_Pack.zip")
for platform, target in (("bizhawk", "rpi4typo"), ("retroarch", "swich")):
with self.subTest(platform=platform), \
contextlib.redirect_stdout(io.StringIO()), \
contextlib.redirect_stderr(io.StringIO()), \
self.assertRaises(SystemExit) as ctx:
self._run(self._args(platform=platform, target=target))
self.assertEqual(ctx.exception.code, 1)
self.assertEqual(self.variants, [])
def test_the_run_ends_on_a_summary(self): def test_the_run_ends_on_a_summary(self):
"""A failing pack was named only above its own error lines, and the """A failing pack was named only above its own error lines, and the
log ended on the last platform's OK line.""" log ended on the last platform's OK line."""
+59
View File
@@ -89,6 +89,12 @@ class ReleaseAssetNotCached(unittest.TestCase):
self.assertEqual(entry["status"], Status.OK) self.assertEqual(entry["status"], Status.OK)
self.assertEqual(entry["release_asset"], REGISTERED[0]) self.assertEqual(entry["release_asset"], REGISTERED[0])
def test_its_path_is_a_pack_destination(self):
"""The pack holds the fetched file at its path: absent from
pack_destinations, step 5 reported it 'pack only'."""
report = verify_platform(self.config, self.db, "emulators", {})
self.assertIn("big.bin", report["pack_destinations"])
def test_offline_counts_it_missing(self): def test_offline_counts_it_missing(self):
packresolve.set_offline(True) packresolve.set_offline(True)
self.assertEqual(self._entry()["status"], Status.MISSING) self.assertEqual(self._entry()["status"], Status.MISSING)
@@ -117,6 +123,59 @@ class ReleaseAssetNotCachedMd5(ReleaseAssetNotCached):
mode = "md5" mode = "md5"
class CoreExtraReleaseAssetNotCached(ReleaseAssetNotCached):
"""The same asset read by a core and declared by no platform entry: the
builder fetched and packed it online, verify's replay of the extras
resolved it offline and left its path out of the pack."""
def setUp(self):
super().setUp()
self.config["systems"]["sys"]["files"] = []
self.config["cores"] = ["bigcore"]
Path("platforms/big.yml").write_text(yaml.dump(self.config))
self.profiles = {"bigcore": {
"emulator": "Big Core", "type": "libretro", "systems": ["sys"],
"files": [{"name": "big.bin", "required": True, "sha1": SHA1}],
}}
Path("emulators/bigcore.yml").write_text(yaml.dump(self.profiles["bigcore"]))
common._platform_config_cache.clear()
common._emulator_profiles_cache.clear()
def _report(self) -> dict:
return verify_platform(self.config, self.db, "emulators", self.profiles)
def test_counted_as_fetched_when_the_pack_is_built(self):
report = self._report()
self.assertIn("big.bin", report["pack_destinations"])
self.assertEqual(report["packed_extras"]["bios"], 1)
def test_its_path_is_a_pack_destination(self):
cache = Path("cache/big.bin")
cache.parent.mkdir()
cache.write_bytes(PAYLOAD)
with mock.patch.object(
packresolve, "fetch_large_file", return_value=str(cache)
):
zip_path = builder.generate_pack(
"big", "platforms", self.db, "bios", "out",
emulators_dir="emulators", emu_profiles=self.profiles, offline=False,
)
with zipfile.ZipFile(zip_path) as zf:
packed = {n for n in zf.namelist() if not n.endswith("/")}
self.assertIn("big.bin", packed)
self.assertIn("big.bin", self._report()["pack_destinations"])
def test_offline_counts_it_missing(self):
packresolve.set_offline(True)
self.assertNotIn("big.bin", self._report()["pack_destinations"])
def test_an_asset_with_other_content_is_not_counted(self):
self.profiles["bigcore"]["files"][0]["sha1"] = "0" * 40
self.assertNotIn("big.bin", self._report()["pack_destinations"])
test_the_online_build_packs_it = None
class OfflinePipelineVerifiesOffline(unittest.TestCase): class OfflinePipelineVerifiesOffline(unittest.TestCase):
"""The offline pipeline builds packs without release assets: its verify """The offline pipeline builds packs without release assets: its verify
step has to count them the same way, or step 5 compares two answers.""" step has to count them the same way, or step 5 compares two answers."""
+2 -1
View File
@@ -117,7 +117,8 @@ RetroDECK: 2007/2008 OK, 1 untested [md5]
The run prints more `DISCREPANCY` lines than this excerpt shows. The core The run prints more `DISCREPANCY` lines than this excerpt shows. The core
and game data lines count what the emulator cores add beyond the platform's and game data lines count what the emulator cores add beyond the platform's
own list. A file held but not packed is left out for the reason in parentheses. own list. A file held but not packed is left out for the reason in parentheses, each
reason with its count when the files have several.
`--verbose` also lists the passing entries that an emulator validates, with `--verbose` also lists the passing entries that an emulator validates, with
the check and the source lines behind it: the check and the source lines behind it:
+2 -1
View File
@@ -134,6 +134,7 @@ python scripts/verify.py --platform recalbox --region us # regional priority l
python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named
python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds
python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build
python scripts/verify.py --list-platforms # list all platforms
python scripts/verify.py --list-emulators # list all emulators python scripts/verify.py --list-emulators # list all emulators
python scripts/verify.py --list-systems # list all systems python scripts/verify.py --list-systems # list all systems
python scripts/verify.py --platform retroarch --list-targets # list available targets python scripts/verify.py --platform retroarch --list-targets # list available targets
@@ -208,7 +209,7 @@ python scripts/generate_pack.py --all --all-variants --output-dir dist/ # all 6
python scripts/generate_pack.py --all --all-variants --verify-packs --output-dir dist/ python scripts/generate_pack.py --all --all-variants --verify-packs --output-dir dist/
# Listing and integrity # Listing and integrity
python scripts/generate_pack.py --list # list available platforms python scripts/generate_pack.py --list-platforms # list available platforms
python scripts/generate_pack.py --list-emulators python scripts/generate_pack.py --list-emulators
python scripts/generate_pack.py --list-systems python scripts/generate_pack.py --list-systems
python scripts/generate_pack.py --platform retroarch --list-targets python scripts/generate_pack.py --platform retroarch --list-targets