mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-06 01:53:56 -06:00
chore: merge audit batch, verify and pack
This commit is contained in:
commit
3f50163b0a
15 files changed
+748
-127
No files matched your search
+64
-1
@@ -664,8 +664,15 @@ def load_target_config(
|
||||
targets_dir = os.path.join(platforms_dir, "targets")
|
||||
target_file = os.path.join(targets_dir, f"{platform_name}.yml")
|
||||
if not os.path.exists(target_file):
|
||||
# The expected state for a platform that ships one build, said as
|
||||
# such: the file path read like a file missing from the repository.
|
||||
with_targets = sorted(
|
||||
Path(f).stem for f in os.listdir(targets_dir)
|
||||
if f.endswith(".yml") and not f.startswith("_")
|
||||
)
|
||||
raise FileNotFoundError(
|
||||
f"No target config for platform '{platform_name}': {target_file}"
|
||||
f"{platform_name} has no hardware targets: drop --target "
|
||||
f"(platforms with targets: {', '.join(with_targets)})"
|
||||
)
|
||||
with open(target_file) as f:
|
||||
data = yaml_load(f) or {}
|
||||
@@ -2986,6 +2993,46 @@ def filter_systems_by_target(
|
||||
}
|
||||
|
||||
|
||||
def platform_scope_systems(
|
||||
config: dict,
|
||||
profiles: dict[str, dict],
|
||||
target_cores: set[str] | None,
|
||||
system_filter: list[str] | None = None,
|
||||
) -> dict[str, dict]:
|
||||
"""The systems a pack or a report of a platform covers.
|
||||
|
||||
Those a core of the target runs, then the ones a --system list keeps.
|
||||
verify.py and generate_pack.py read this one function.
|
||||
"""
|
||||
plat_cores = resolve_platform_cores(config, profiles) if target_cores else None
|
||||
systems = filter_systems_by_target(
|
||||
config.get("systems", {}), profiles, target_cores, platform_cores=plat_cores
|
||||
)
|
||||
return select_systems(systems, system_filter) if system_filter else systems
|
||||
|
||||
|
||||
def system_scope_error(
|
||||
config: dict, platform_name: str, system_filter: list[str], target: str | None
|
||||
) -> str:
|
||||
"""Why a --system list leaves a platform no system.
|
||||
|
||||
A system the platform declares but no core of the target runs was
|
||||
reported as a system the platform lacks, with a lookalike to try and a
|
||||
listing that refuses --target.
|
||||
"""
|
||||
systems = config.get("systems", {})
|
||||
declared = select_systems(systems, system_filter)
|
||||
if declared and target:
|
||||
return (
|
||||
f"no core {platform_name} runs on target {target} emulates "
|
||||
f"{', '.join(sorted(declared))}"
|
||||
)
|
||||
hint = suggest_names(
|
||||
system_filter, sorted(systems), f"--platform {platform_name} --list-systems"
|
||||
)
|
||||
return f"{platform_name} has no system matching {', '.join(system_filter)}. {hint}"
|
||||
|
||||
|
||||
def platform_declarations(
|
||||
config: dict, db: dict, *, enrich: bool = True
|
||||
) -> dict[str, list[tuple[int | None, str]]]:
|
||||
@@ -3145,6 +3192,22 @@ def list_system_ids(emulators_dir: str) -> None:
|
||||
print(f" {spelling:35s} ({count} emulator{'s' if count > 1 else ''})")
|
||||
|
||||
|
||||
def print_platform_listing(platforms_dir: str) -> None:
|
||||
"""Every registered platform, one per line, marked when archived or
|
||||
served by the installer only.
|
||||
|
||||
Every platform takes --platform. --all covers the active ones, and the
|
||||
pack builder's --all the release packs. Printed by --list-platforms in
|
||||
verify.py and generate_pack.py, the name install.py gives it.
|
||||
"""
|
||||
active = set(list_registered_platforms(platforms_dir))
|
||||
released = set(list_release_pack_platforms(platforms_dir, include_archived=True))
|
||||
for name in list_registered_platforms(platforms_dir, include_archived=True):
|
||||
marks = [m for m, on in (("archived", name not in active),
|
||||
("installer only", name not in released)) if on]
|
||||
print(f"{name} ({', '.join(marks)})" if marks else name)
|
||||
|
||||
|
||||
def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None:
|
||||
"""Print system IDs from a platform's YAML config, with the platform's
|
||||
own names for them, which --system also accepts."""
|
||||
|
||||
+107
-72
@@ -53,7 +53,10 @@ from common import (
|
||||
list_platform_system_ids,
|
||||
platform_system_names,
|
||||
resolve_system_arg,
|
||||
print_platform_listing,
|
||||
select_systems,
|
||||
platform_scope_systems,
|
||||
system_scope_error,
|
||||
canonical_target_name,
|
||||
platform_files_target,
|
||||
list_registered_platforms,
|
||||
@@ -753,31 +756,13 @@ def generate_pack(
|
||||
target_cores=target_cores, source=source,
|
||||
)
|
||||
|
||||
# Filter systems by target if specified
|
||||
plat_cores = (
|
||||
resolve_platform_cores(config, emu_profiles or {}) if target_cores else None
|
||||
pack_systems = platform_scope_systems(
|
||||
config, emu_profiles or {}, target_cores, system_filter
|
||||
)
|
||||
pack_systems = filter_systems_by_target(
|
||||
config.get("systems", {}),
|
||||
emu_profiles or {},
|
||||
target_cores,
|
||||
platform_cores=plat_cores,
|
||||
)
|
||||
|
||||
if system_filter:
|
||||
filtered = select_systems(pack_systems, system_filter)
|
||||
if not filtered:
|
||||
hint = suggest_names(
|
||||
system_filter,
|
||||
sorted(pack_systems),
|
||||
f"--platform {platform_name} --list-systems",
|
||||
)
|
||||
print(
|
||||
f" WARNING: {platform_name} has no system matching "
|
||||
f"{', '.join(system_filter)}. {hint}"
|
||||
)
|
||||
return None
|
||||
pack_systems = filtered
|
||||
if system_filter and not pack_systems:
|
||||
reason = system_scope_error(config, platform_name, system_filter, target_name)
|
||||
print(f" WARNING: {reason}")
|
||||
return None
|
||||
if extras_only:
|
||||
# The part of a split that carries the core extras no platform system
|
||||
# owns: the platform's own files and trees went to the other parts.
|
||||
@@ -1882,11 +1867,6 @@ def generate_system_pack(
|
||||
shutil.rmtree(scratch, ignore_errors=True)
|
||||
|
||||
|
||||
def list_platforms(platforms_dir: str) -> list[str]:
|
||||
"""List available platform names from registry."""
|
||||
return list_registered_platforms(platforms_dir, include_archived=True)
|
||||
|
||||
|
||||
def _system_display_name(system_id: str) -> str:
|
||||
"""Convert system ID to display name for ZIP naming."""
|
||||
s = system_id.lower().replace("_", "-")
|
||||
@@ -1935,7 +1915,6 @@ def generate_split_packs(
|
||||
) -> list[str]:
|
||||
"""Generate split packs (one ZIP per system or manufacturer)."""
|
||||
config = load_platform_config(platform_name, platforms_dir)
|
||||
platform_display = config.get("platform", platform_name)
|
||||
# The split directory carries the same tags as the packs inside it, or two
|
||||
# differently narrowed runs write into one another.
|
||||
split_tags = "".join(
|
||||
@@ -1951,13 +1930,19 @@ def generate_split_packs(
|
||||
# The parts carry the version, so the directory does too: a rescrape that
|
||||
# moves the version would otherwise leave the old parts beside the new
|
||||
# under one SHA256SUMS.txt.
|
||||
version = config.get("version", config.get("dat_version", ""))
|
||||
ver_tag = f"_{version.replace(' ', '')}" if version else ""
|
||||
part_prefix = f"{platform_display.replace(' ', '_')}{ver_tag}{split_tags}"
|
||||
# Named by the stem the full pack reads: assembled here by hand, a '/' in
|
||||
# a version nested the directory where the check never looked for it.
|
||||
part_prefix = (
|
||||
f"{_platform_pack_stem([platform_name], platform_name, platforms_dir)}{split_tags}"
|
||||
)
|
||||
split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split")
|
||||
os.makedirs(split_dir, exist_ok=True)
|
||||
|
||||
systems = config.get("systems", {})
|
||||
if emu_profiles is None:
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
# The systems a core of the target runs: a part per declared system sent
|
||||
# each one the target drops to a refusal naming it as unknown.
|
||||
systems = platform_scope_systems(config, emu_profiles, target_cores)
|
||||
|
||||
if group_by == "manufacturer":
|
||||
groups = _group_systems_by_manufacturer(systems, db, bios_dir)
|
||||
@@ -1966,8 +1951,6 @@ def generate_split_packs(
|
||||
|
||||
# Pre-compute core extras once (expensive: scans 260+ emulator profiles)
|
||||
# then distribute per group based on emulator system overlap
|
||||
if emu_profiles is None:
|
||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||
base_dest = config.get("base_destination", "")
|
||||
# Each part cuts the platform's own entries from the full profiles. The
|
||||
# extras are collected once here, already without the cores' game data.
|
||||
@@ -2373,6 +2356,46 @@ def _resolve_systems(args: argparse.Namespace, parser: argparse.ArgumentParser)
|
||||
parser.error(f"--system {args.system}: {exc}")
|
||||
|
||||
|
||||
def _platforms_with_systems(
|
||||
args: argparse.Namespace,
|
||||
platforms: list[str],
|
||||
system_filter: list[str],
|
||||
emu_profiles: dict,
|
||||
target_cores_cache: dict[str, set[str] | None],
|
||||
) -> list[str]:
|
||||
"""The platforms a --system list leaves a system to pack, before any lock.
|
||||
|
||||
Under --all a platform without one is out of scope, as a platform
|
||||
without the target is: `--all --system sony-playstation-2` built every
|
||||
PS2 pack and exited 1 on MiSTer, which has no PS2. Under --platform the
|
||||
request is refused here, with its reason.
|
||||
"""
|
||||
kept = []
|
||||
for name in platforms:
|
||||
config = load_platform_config(name, args.platforms_dir)
|
||||
tc = target_cores_cache.get(name) if args.target else None
|
||||
if platform_scope_systems(config, emu_profiles, tc, system_filter):
|
||||
kept.append(name)
|
||||
continue
|
||||
if not args.all:
|
||||
reason = system_scope_error(config, name, system_filter, args.target)
|
||||
print(f"ERROR: {reason}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
declared = select_systems(config.get("systems", {}), system_filter)
|
||||
reason = (
|
||||
system_scope_error(config, name, system_filter, args.target)
|
||||
if declared and args.target
|
||||
else f"no system {', '.join(system_filter)}"
|
||||
)
|
||||
print(f"INFO: Skipping {name}: {reason}")
|
||||
if not kept:
|
||||
where = f" on target {args.target}" if args.target else ""
|
||||
print(f"ERROR: no platform packed here runs {', '.join(system_filter)}{where}",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
return kept
|
||||
|
||||
|
||||
def _refuse_for_verify_packs(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
||||
"""What --verify-packs cannot be combined with, refused before any lock.
|
||||
|
||||
@@ -2652,6 +2675,19 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
|
||||
else:
|
||||
raise ValueError("--verify-packs requires --platform or --all")
|
||||
|
||||
# The target is checked as the build checks it: a mistyped one, or one
|
||||
# the platform does not publish, was dropped and the full pack verified
|
||||
# under it, `all OK` for a pack no build had made.
|
||||
target_cores_cache: dict[str, set[str] | None] = {}
|
||||
if args.target:
|
||||
try:
|
||||
target_cores_cache, platforms = build_target_cores_cache(
|
||||
platforms, args.target, args.platforms_dir, is_all=args.all
|
||||
)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
all_ok = True
|
||||
verify_regions = getattr(args, "regions", None)
|
||||
verify_profiles = load_emulator_profiles(args.emulators_dir)
|
||||
@@ -2689,7 +2725,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
|
||||
else:
|
||||
print(f" {platform_name}: SKIP (no pack in {args.output_dir})")
|
||||
continue
|
||||
tc = _target_cores_for(platform_name, args.target, args.platforms_dir)
|
||||
tc = target_cores_cache.get(platform_name)
|
||||
# A group pack and a pack built for the platform alone can both be
|
||||
# present, and each one is a pack of this platform.
|
||||
for pack_name in found:
|
||||
@@ -3057,7 +3093,7 @@ def _refuse_unapplied_flags(args: argparse.Namespace, parser: argparse.ArgumentP
|
||||
name for name, on in (
|
||||
("--manifest-targets", args.manifest_targets),
|
||||
("--verify-packs", args.verify_packs and not args.all_variants),
|
||||
("--list", args.list),
|
||||
("--list-platforms", args.list),
|
||||
("--list-emulators", args.list_emulators),
|
||||
("--list-systems", args.list_systems),
|
||||
("--list-targets", args.list_targets),
|
||||
@@ -3083,7 +3119,7 @@ QUICK_MODE_FLAGS: dict[str, set[str]] = {
|
||||
},
|
||||
# One manifest per hardware target, from the target files themselves.
|
||||
"--manifest-targets": set(),
|
||||
"--list": set(),
|
||||
"--list-platforms": set(),
|
||||
"--list-emulators": set(),
|
||||
"--list-systems": {"--platform"},
|
||||
"--list-targets": {"--platform"},
|
||||
@@ -3112,7 +3148,7 @@ def _given_flags(args: argparse.Namespace) -> set[str]:
|
||||
"--target": args.target,
|
||||
"--region": args.region,
|
||||
"--one-per-slot": args.one_per_slot,
|
||||
"--list": args.list,
|
||||
"--list-platforms": args.list,
|
||||
"--list-emulators": args.list_emulators,
|
||||
"--list-systems": args.list_systems,
|
||||
"--list-targets": args.list_targets,
|
||||
@@ -3209,7 +3245,11 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
action="store_true",
|
||||
help="Pack the data directories as cached, still fetching release assets",
|
||||
)
|
||||
parser.add_argument("--list", action="store_true", help="List available platforms")
|
||||
# install.py's name. --list stays for the scripts that already type it.
|
||||
parser.add_argument(
|
||||
"--list-platforms", "--list", dest="list", action="store_true",
|
||||
help="List available platforms",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--required-only",
|
||||
action="store_true",
|
||||
@@ -3311,14 +3351,7 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
)
|
||||
return
|
||||
if args.list:
|
||||
# Every platform builds with --platform. --all builds the release
|
||||
# packs only, the others are served by the installer.
|
||||
active = set(list_registered_platforms(args.platforms_dir))
|
||||
released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True))
|
||||
for p in list_platforms(args.platforms_dir):
|
||||
marks = [m for m, on in (("archived", p not in active),
|
||||
("installer only", p not in released)) if on]
|
||||
print(f"{p} ({', '.join(marks)})" if marks else p)
|
||||
print_platform_listing(args.platforms_dir)
|
||||
return
|
||||
if args.list_emulators:
|
||||
list_emulator_profiles(args.emulators_dir)
|
||||
@@ -3464,6 +3497,28 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
parser.error("Specify --platform or --all")
|
||||
return
|
||||
|
||||
# A mistyped target is refused before the output lock and the data
|
||||
# directory refresh: checked after them, `--target swich` waited on a
|
||||
# pipeline's caches before saying the target does not exist.
|
||||
target_cores_cache: dict[str, set[str] | None] = {}
|
||||
if args.target:
|
||||
try:
|
||||
target_cores_cache, platforms = build_target_cores_cache(
|
||||
platforms,
|
||||
args.target,
|
||||
args.platforms_dir,
|
||||
is_all=args.all,
|
||||
)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
emu_profiles = load_emulator_profiles(args.emulators_dir)
|
||||
if system_filter:
|
||||
platforms = _platforms_with_systems(
|
||||
args, platforms, system_filter, emu_profiles, target_cores_cache
|
||||
)
|
||||
|
||||
data_registry = load_data_dir_registry(args.platforms_dir)
|
||||
# The output directory is claimed before the data directory refresh: a
|
||||
# refresh waits for every run that holds a cache, so a second build
|
||||
@@ -3483,21 +3538,6 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
if data_registry:
|
||||
hold_data_dirs(held, data_registry)
|
||||
|
||||
emu_profiles = load_emulator_profiles(args.emulators_dir)
|
||||
|
||||
target_cores_cache: dict[str, set[str] | None] = {}
|
||||
if args.target:
|
||||
try:
|
||||
target_cores_cache, platforms = build_target_cores_cache(
|
||||
platforms,
|
||||
args.target,
|
||||
args.platforms_dir,
|
||||
is_all=args.all,
|
||||
)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
groups = group_identical_platforms(
|
||||
platforms, args.platforms_dir, target_cores_cache if args.target else None
|
||||
)
|
||||
@@ -4877,17 +4917,12 @@ def _target_cores_for(
|
||||
) -> set[str] | None:
|
||||
"""Cores a target leaves available, or None when no target was asked for.
|
||||
|
||||
None means no narrowing, which is what an unfiltered pack expects. Read
|
||||
the way --all builds: a platform without a target file was packed
|
||||
unnarrowed under the target's name, and checking that pack must not
|
||||
raise where building it did not.
|
||||
Called on a platform the build already packed under the target, so the
|
||||
target is known to it: a platform without one is left out of the run.
|
||||
"""
|
||||
if not target_name:
|
||||
return None
|
||||
try:
|
||||
return load_target_config(platform_name, target_name, platforms_dir)
|
||||
except (FileNotFoundError, ValueError):
|
||||
return None
|
||||
return load_target_config(platform_name, target_name, platforms_dir)
|
||||
|
||||
|
||||
def _expected_pack_names(
|
||||
|
||||
+14
-1
@@ -244,6 +244,7 @@ class ExtraResolver:
|
||||
offline: bool | None = True,
|
||||
*,
|
||||
profiles: dict,
|
||||
pending_assets: bool = False,
|
||||
) -> None:
|
||||
self.profiles = profiles
|
||||
self.db = db
|
||||
@@ -253,10 +254,13 @@ class ExtraResolver:
|
||||
self.validation_index = validation_index
|
||||
self.validation_owners = validation_owners
|
||||
self.offline = offline
|
||||
# verify's replay: never downloads, and counts the release asset an
|
||||
# online build fetches, as it does for a platform entry.
|
||||
self.pending_assets = pending_assets
|
||||
|
||||
def resolve(self, fe: dict, dest: str) -> tuple[str | None, str, str]:
|
||||
"""(local copy or None, resolver status, the core's disagreement)."""
|
||||
from packresolve import resolve_file
|
||||
from packresolve import pending_release_asset, resolve_file, run_is_offline
|
||||
from validation import core_size_refusal, extra_alternatives, validated_choice
|
||||
|
||||
local_path, status = resolve_file(
|
||||
@@ -264,6 +268,15 @@ class ExtraResolver:
|
||||
dest_hint=fe.get("destination", ""),
|
||||
data_dir_registry=self.data_registry, offline=self.offline,
|
||||
)
|
||||
if (
|
||||
self.pending_assets and status in ("not_found", "hash_mismatch")
|
||||
and not run_is_offline()
|
||||
):
|
||||
# The asset the online builder fetches in resolve_file. Its record
|
||||
# answers the declaration, and there are no bytes on disk to check.
|
||||
asset = pending_release_asset(fe, self.db)
|
||||
if asset:
|
||||
return asset, "release_asset", ""
|
||||
if status in ("not_found", "external", "user_provided") or not local_path:
|
||||
return None, status, ""
|
||||
local_path, disagreement = validated_choice(
|
||||
|
||||
+24
-9
@@ -30,7 +30,7 @@ from packextras import platform_region_groups
|
||||
from packpaths import _flat
|
||||
from nativemode import reads_file_contents
|
||||
import region as region_mod
|
||||
from packresolve import resolve_file
|
||||
from packresolve import resolve_declaration, resolve_file
|
||||
from exporter.candidates import BIOS, EntryCategories
|
||||
from gamedata import without_game_data
|
||||
from common import resolve_platform_cores
|
||||
@@ -544,6 +544,21 @@ def _structural_errors(zf: zipfile.ZipFile, zip_set: set) -> list[str]:
|
||||
return errors
|
||||
|
||||
|
||||
def _shipped_copy(
|
||||
entry: dict, db: dict, data_registry: dict | None, mode: str, hash_rule: str,
|
||||
) -> str | None:
|
||||
"""The file the builder packs for a declaration, or None.
|
||||
|
||||
The verdict verify applies before it lets a declaration hold its path.
|
||||
"""
|
||||
local, status = resolve_declaration(
|
||||
entry, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True,
|
||||
)
|
||||
if local and baseline_verdict(entry, local, status, mode, hash_rule)[1] is None:
|
||||
return local
|
||||
return None
|
||||
|
||||
|
||||
def _builder_extras(
|
||||
config: dict,
|
||||
emulators_dir: str,
|
||||
@@ -563,15 +578,17 @@ def _builder_extras(
|
||||
"""{full destination: local copy} of each core extra the builder writes.
|
||||
|
||||
The platform's paths come first, as in the builder: a destination the
|
||||
pack holds whose declarations resolve to a file, or that the platform
|
||||
keeps for a user_provided or external entry. One whose declarations
|
||||
resolve to nothing is left free, since the builder then places a core's
|
||||
file there.
|
||||
pack holds whose declarations resolve to a file it ships, or that the
|
||||
platform keeps for a user_provided or external entry. One whose
|
||||
declarations resolve to nothing, or to a file the frontend's hash
|
||||
refuses, is left free, since the builder then places a core's file
|
||||
there: taken by the refused file, that core file was never compared.
|
||||
"""
|
||||
from packextras import ExtraResolver, hidden_extras
|
||||
from packresolve import resolve_declaration
|
||||
from validation import _build_validation_index, destination_owners
|
||||
|
||||
mode = config.get("verification_mode", "existence")
|
||||
hash_rule = config.get("hash_rule", "")
|
||||
shipped: dict[str, str | None] = {}
|
||||
for expected, declarations in baseline_groups.items():
|
||||
if expected not in zip_set and expected.lower() not in zip_lower:
|
||||
@@ -587,9 +604,7 @@ def _builder_extras(
|
||||
local = next(
|
||||
(
|
||||
found for fe in declarations
|
||||
if (found := resolve_declaration(
|
||||
fe, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True,
|
||||
)[0])
|
||||
if (found := _shipped_copy(fe, db, data_registry, mode, hash_rule))
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
+23
-5
@@ -100,6 +100,20 @@ class Conflict:
|
||||
def emulators(self) -> list[str]:
|
||||
return sorted({c.emulator for c in self.profile_claims if c.emulator})
|
||||
|
||||
def files_read(self) -> list[tuple[str | None, list[str], str]]:
|
||||
"""Each file the profiles want here, with the emulators reading it
|
||||
and the proof of the first claim on it.
|
||||
|
||||
One file stood for every claim: Recalbox's neocdz.zip line gave
|
||||
geolith the archive fbneo reads, not the one geolith pins.
|
||||
"""
|
||||
files: dict[str | None, tuple[list[str], str]] = {}
|
||||
for claim in self.profile_claims:
|
||||
emulators, status = files.setdefault(claim.local_path, ([], claim.status))
|
||||
if claim.emulator and claim.emulator not in emulators:
|
||||
emulators.append(claim.emulator)
|
||||
return [(path, sorted(emus), status) for path, (emus, status) in files.items()]
|
||||
|
||||
|
||||
def _normalize(destination: str, fold: bool = True) -> str:
|
||||
"""Comparable form of a destination, since layers write it differently.
|
||||
@@ -661,11 +675,13 @@ def format_decision(decision: Decision) -> str:
|
||||
f"{conflict.destination}: serve {decision.winner.local_path}, "
|
||||
"the pack answers to the emulator"
|
||||
)
|
||||
expected = ", ".join(conflict.emulators) or "the profile"
|
||||
rejected = ", ".join(
|
||||
f"{path}, the file {', '.join(emulators) or 'the profile'} reads"
|
||||
for path, emulators, _status in conflict.files_read()
|
||||
)
|
||||
return (
|
||||
f"{conflict.destination}: keep {decision.winner.local_path}, the frontend "
|
||||
f"verifies content and would reject "
|
||||
f"{conflict.profile_claims[0].local_path}, the file {expected} reads"
|
||||
f"verifies content and would reject {rejected}"
|
||||
)
|
||||
|
||||
|
||||
@@ -996,8 +1012,10 @@ def main() -> int:
|
||||
"ships": c.platform_claim.local_path,
|
||||
"ships_evidence": c.platform_claim.status,
|
||||
"emulators": c.emulators,
|
||||
"expected": c.profile_claims[0].local_path,
|
||||
"expected_evidence": c.profile_claims[0].status,
|
||||
"expected": [
|
||||
{"file": path, "evidence": status, "emulators": emulators}
|
||||
for path, emulators, status in c.files_read()
|
||||
],
|
||||
}
|
||||
for c in conflicts
|
||||
]
|
||||
|
||||
+78
-33
@@ -29,6 +29,7 @@ import contextlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
|
||||
@@ -54,7 +55,6 @@ from common import (
|
||||
compute_hashes,
|
||||
expand_directory_entries,
|
||||
platform_declarations,
|
||||
filter_systems_by_target,
|
||||
group_identical_platforms,
|
||||
list_emulator_profiles,
|
||||
list_system_ids,
|
||||
@@ -63,7 +63,10 @@ from common import (
|
||||
load_emulator_profiles,
|
||||
load_platform_config,
|
||||
resolve_system_arg,
|
||||
print_platform_listing,
|
||||
select_systems,
|
||||
platform_scope_systems,
|
||||
system_scope_error,
|
||||
md5sum,
|
||||
require_yaml,
|
||||
resolve_local_file,
|
||||
@@ -1217,16 +1220,7 @@ def verify_platform(
|
||||
validation_index = _build_validation_index(platform_profiles)
|
||||
validation_owners = destination_owners(platform_profiles)
|
||||
|
||||
if not target_cores:
|
||||
plat_cores = None
|
||||
verify_systems = filter_systems_by_target(
|
||||
config.get("systems", {}),
|
||||
profiles,
|
||||
target_cores,
|
||||
platform_cores=plat_cores,
|
||||
)
|
||||
if system_filter:
|
||||
verify_systems = select_systems(verify_systems, system_filter)
|
||||
verify_systems = platform_scope_systems(config, profiles, target_cores, system_filter)
|
||||
|
||||
details = []
|
||||
file_status: dict[str, str] = {}
|
||||
@@ -1315,8 +1309,12 @@ def verify_platform(
|
||||
"release_asset": local_path,
|
||||
"reason": "release asset, fetched when the pack is built",
|
||||
}
|
||||
# Written at its path like any held file: keyed to None, it
|
||||
# stayed out of pack_destinations and step 5 read the pack's
|
||||
# copy as a file only the pack holds.
|
||||
packed_order.append((
|
||||
sys_id, f"{base_dest}/{destination}" if base_dest else destination, None,
|
||||
sys_id, f"{base_dest}/{destination}" if base_dest else destination,
|
||||
local_path,
|
||||
))
|
||||
shipped_keys.add(_dest_key(file_entry, sys_id))
|
||||
local_path = None
|
||||
@@ -1465,7 +1463,7 @@ def verify_platform(
|
||||
region_drops, report_for_builder,
|
||||
ExtraResolver(
|
||||
db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index,
|
||||
validation_owners, profiles=profiles,
|
||||
validation_owners, profiles=profiles, pending_assets=True,
|
||||
),
|
||||
placed_kind,
|
||||
system_filter,
|
||||
@@ -1746,13 +1744,20 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) ->
|
||||
print(f" {line}")
|
||||
|
||||
|
||||
def _print_undeclared_entry(u: dict, prefix: str, verbose: bool) -> None:
|
||||
"""Print a single undeclared file entry with its validation checks."""
|
||||
def _print_undeclared_entry(
|
||||
u: dict, prefix: str, verbose: bool, repeated: frozenset[str] = frozenset()
|
||||
) -> None:
|
||||
"""Print a single undeclared file entry with its validation checks.
|
||||
|
||||
A name in ``repeated`` is read at more than one path, and the line names
|
||||
the path: two identical lines told no one there were two slots to fill.
|
||||
"""
|
||||
name = u["path"] if u["name"] in repeated and u.get("path") else u["name"]
|
||||
arc_count = u.get("archive_file_count")
|
||||
if arc_count:
|
||||
name_label = f"{u['name']} ({arc_count} file{'s' if arc_count != 1 else ''})"
|
||||
name_label = f"{name} ({arc_count} file{'s' if arc_count != 1 else ''})"
|
||||
else:
|
||||
name_label = u["name"]
|
||||
name_label = name
|
||||
print(f" {prefix}: {u['emulator']} needs {name_label}")
|
||||
checks = u.get("checks", [])
|
||||
if checks:
|
||||
@@ -1800,6 +1805,25 @@ def _core_required_missing(bios_files: list[dict]) -> list[dict]:
|
||||
]
|
||||
|
||||
|
||||
def _repeated_names(entries: list[dict]) -> frozenset[str]:
|
||||
"""Names more than one of ``entries`` carries."""
|
||||
counts = Counter(u["name"] for u in entries)
|
||||
return frozenset(name for name, count in counts.items() if count > 1)
|
||||
|
||||
|
||||
def _omission_reasons(entries: list[dict]) -> str:
|
||||
"""Each reason a held file was left out, with its count when several.
|
||||
|
||||
Printing the first entry's reason for the whole count named one cause
|
||||
for 412 RomM files when 20 carried it.
|
||||
"""
|
||||
counts = Counter(u.get("omitted", "not placed") for u in entries)
|
||||
if len(counts) == 1:
|
||||
return next(iter(counts))
|
||||
ranked = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
|
||||
return ", ".join(f"{count} {reason}" for reason, count in ranked)
|
||||
|
||||
|
||||
def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
"""Print cross-reference section for undeclared files used by cores."""
|
||||
undeclared = result.get("undeclared_files", [])
|
||||
@@ -1839,19 +1863,22 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, "
|
||||
f"{core_missing_opt} optional missing{listed}"
|
||||
)
|
||||
# The missing files follow the line that counts them: printed after the
|
||||
# held-but-not-packed line, they read as files installed with their emulator.
|
||||
repeated = _repeated_names(req_not_in_repo + req_hle_not_in_repo + opt_not_in_repo)
|
||||
for u in req_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (required)", verbose, repeated)
|
||||
for u in req_hle_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose, repeated)
|
||||
if verbose:
|
||||
for u in opt_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (optional)", verbose, repeated)
|
||||
if bundled:
|
||||
names = ", ".join(sorted({u["name"] for u in bundled}))
|
||||
print(f" Bundled with their emulator, not collected: {names}")
|
||||
if unplaced:
|
||||
print(f" Core files held but not packed: {len(unplaced)} ({unplaced[0]['omitted']})")
|
||||
|
||||
for u in req_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (required)", verbose)
|
||||
for u in req_hle_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose)
|
||||
if verbose:
|
||||
for u in opt_not_in_repo:
|
||||
_print_undeclared_entry(u, "MISSING (optional)", verbose)
|
||||
print(f" Core files held but not packed: {len(unplaced)} "
|
||||
f"({_omission_reasons(unplaced)})")
|
||||
|
||||
if game_data:
|
||||
gd_missing = [u for u in game_data if not u["in_repo"]]
|
||||
@@ -1864,12 +1891,13 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||
listed = "" if verbose or not gd_missing else " (--verbose lists them)"
|
||||
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}")
|
||||
if verbose:
|
||||
repeated = _repeated_names(gd_missing)
|
||||
for u in gd_missing:
|
||||
_print_undeclared_entry(u, "MISSING (game data)", verbose)
|
||||
_print_undeclared_entry(u, "MISSING (game data)", verbose, repeated)
|
||||
if gd_unplaced:
|
||||
print(
|
||||
f" Game data held but not packed: {len(gd_unplaced)} "
|
||||
f"({gd_unplaced[0].get('omitted', 'not placed')})"
|
||||
f"({_omission_reasons(gd_unplaced)})"
|
||||
)
|
||||
|
||||
|
||||
@@ -2597,6 +2625,7 @@ def _refuse_listing_narrowings(
|
||||
"""
|
||||
listing = next(
|
||||
(flag for flag, on in (
|
||||
("--list-platforms", args.list_platforms),
|
||||
("--list-emulators", args.list_emulators),
|
||||
("--list-systems", args.list_systems),
|
||||
("--list-targets", args.list_targets),
|
||||
@@ -2605,7 +2634,7 @@ def _refuse_listing_narrowings(
|
||||
)
|
||||
if not listing:
|
||||
return
|
||||
reads_platform = listing != "--list-emulators"
|
||||
reads_platform = listing not in ("--list-emulators", "--list-platforms")
|
||||
for flag, on in (
|
||||
("--platform", args.platform and not reads_platform),
|
||||
("--all", args.all),
|
||||
@@ -2655,7 +2684,9 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
|
||||
|
||||
|
||||
def _run_listing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
||||
if args.list_emulators:
|
||||
if args.list_platforms:
|
||||
print_platform_listing(args.platforms_dir)
|
||||
elif args.list_emulators:
|
||||
list_emulator_profiles(args.emulators_dir)
|
||||
elif args.list_systems and args.platform:
|
||||
list_platform_system_ids(args.platform, args.platforms_dir)
|
||||
@@ -2694,6 +2725,9 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
parser.add_argument(
|
||||
"--list-systems", action="store_true", help="List available systems"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list-platforms", action="store_true", help="List available platforms"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--include-archived", action="store_true",
|
||||
help="With --all, also verify the archived platforms (RetroPie)",
|
||||
@@ -2767,7 +2801,7 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
parser.error(str(exc))
|
||||
|
||||
_refuse_listing_narrowings(args, parser)
|
||||
if args.list_emulators or args.list_systems or args.list_targets:
|
||||
if args.list_platforms or args.list_emulators or args.list_systems or args.list_targets:
|
||||
_run_listing(args, parser)
|
||||
return
|
||||
|
||||
@@ -2853,6 +2887,18 @@ def _main(held: contextlib.ExitStack) -> None:
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"ERROR: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
# A system the target leaves no core for was reported 0/0 with exit 0,
|
||||
# where generate_pack refuses the same request.
|
||||
if platform_systems:
|
||||
config = load_platform_config(args.platform, args.platforms_dir)
|
||||
if not platform_scope_systems(
|
||||
config, emu_profiles, target_cores_cache.get(args.platform), platform_systems
|
||||
):
|
||||
reason = system_scope_error(
|
||||
config, args.platform, platform_systems, args.target
|
||||
)
|
||||
print(f"ERROR: {reason}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Group identical platforms (same function as generate_pack)
|
||||
groups = group_identical_platforms(
|
||||
@@ -2961,8 +3007,7 @@ def _overall_summary(group_results: list[tuple[dict, list[str]]]) -> str:
|
||||
platforms = sum(len(group) for _, group in group_results)
|
||||
|
||||
def where(counts: dict[str, int]) -> str:
|
||||
named = [f"{name} {count}" if count > 1 else name
|
||||
for name, count in counts.items() if count]
|
||||
named = [f"{name} {count}" for name, count in counts.items() if count]
|
||||
return f" ({', '.join(named)})" if named else ""
|
||||
|
||||
parts = []
|
||||
|
||||
@@ -4830,7 +4830,7 @@ class VerifyAllSummaryNamesWhatIsWrong(unittest.TestCase):
|
||||
summary = verify._overall_summary(
|
||||
[(clean, ["RetroArch"]), (warned, ["RetroDECK"]), (core_miss, ["ROCKNIX"])]
|
||||
)
|
||||
self.assertIn("1 warning (RetroDECK)", summary)
|
||||
self.assertIn("1 warning (RetroDECK 1)", summary)
|
||||
self.assertIn("ROCKNIX: n64.psh", summary)
|
||||
self.assertIn("0 critical,", summary)
|
||||
|
||||
|
||||
@@ -87,6 +87,59 @@ class CoreFilesAsTheBuilderPlacesThem(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class ARefusedPinLeavesItsPathToTheCore(unittest.TestCase):
|
||||
"""EmuELEC pins naomi.zip by an md5 the collection does not hold: the
|
||||
builder refuses the held copy and ships the core's file at that path.
|
||||
The check marked the path taken by the refused platform file, so the
|
||||
core file the pack carries there was never compared."""
|
||||
|
||||
def setUp(self):
|
||||
tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
|
||||
self.addCleanup(tmp.cleanup)
|
||||
previous = os.getcwd()
|
||||
os.chdir(tmp.name)
|
||||
self.addCleanup(os.chdir, previous)
|
||||
files: dict = {}
|
||||
by_name: dict = {}
|
||||
_held(files, by_name, "bios/dc/naomi.zip", b"the core's naomi")
|
||||
self.db = {"files": files, "indexes": {
|
||||
"by_name": by_name, "by_md5": {e["md5"]: s for s, e in files.items()},
|
||||
"by_path_suffix": {}, "by_crc32": {}, "by_sha256": {}}}
|
||||
Path("platforms").mkdir()
|
||||
config = {"platform": "P", "verification_mode": "md5", "cores": ["flycast"],
|
||||
"systems": {"dc": {"files": [{
|
||||
"name": "naomi.zip", "destination": "dc/naomi.zip",
|
||||
"md5": hashlib.md5(b"a bundle nobody holds").hexdigest(),
|
||||
}]}}}
|
||||
Path("platforms/p.yml").write_text(yaml.safe_dump(config), encoding="utf-8")
|
||||
Path("emulators").mkdir()
|
||||
self.profiles = {"flycast": {
|
||||
"emulator": "flycast", "type": "libretro", "systems": ["dc"], "files": [
|
||||
{"name": "naomi.zip", "path": "dc/naomi.zip"}]}}
|
||||
|
||||
def _check(self, member: bytes) -> tuple:
|
||||
with zipfile.ZipFile("pack.zip", "w") as zf:
|
||||
zf.writestr("dc/naomi.zip", member)
|
||||
return packverify.verify_pack_against_platform(
|
||||
"pack.zip", "p", "platforms", self.db, emulators_dir="emulators",
|
||||
emu_profiles=self.profiles,
|
||||
)
|
||||
|
||||
def test_the_core_file_at_the_refused_path_is_compared(self):
|
||||
_ok, _c, _p, errors, *_mid, core_checked, _cp, _ex, _ab = self._check(b"other")
|
||||
self.assertEqual(core_checked, 1)
|
||||
self.assertTrue(
|
||||
any(e.startswith("core file differs: dc/naomi.zip") for e in errors), errors
|
||||
)
|
||||
|
||||
def test_the_right_core_file_passes(self):
|
||||
_ok, _c, _p, errors, *_mid, core_checked, core_present, _ex, _ab = (
|
||||
self._check(b"the core's naomi")
|
||||
)
|
||||
self.assertEqual((core_checked, core_present), (1, 1))
|
||||
self.assertFalse([e for e in errors if e.startswith("core")], errors)
|
||||
|
||||
|
||||
class NarrowedPacksAgainstTheirOwnList(unittest.TestCase):
|
||||
"""--verify-packs after --all-variants said "hashes checked during the
|
||||
build" for the five narrowed packs and compared none of them: a narrowed
|
||||
|
||||
@@ -420,6 +420,24 @@ class TestArbitration(unittest.TestCase):
|
||||
served = slots.format_decision(slots.arbitrate(self._conflict(), "existence"))
|
||||
self.assertIn("both are satisfied", served)
|
||||
|
||||
def test_each_emulator_is_given_the_file_it_reads(self):
|
||||
"""Recalbox's neocdz.zip line said geolith reads the archive fbneo
|
||||
reads, while geolith pins another one."""
|
||||
conflict = self._conflict()
|
||||
conflict.profile_claims = [
|
||||
slots.Claim("profile", "neocdz.zip", "neocdz.zip", emulator="fbneo",
|
||||
local_path="a/neocdz.zip.ba20a7be", status="members_held"),
|
||||
slots.Claim("profile", "neocdz.zip", "neocdz.zip", emulator="geolith",
|
||||
local_path="b/neocdz.zip.712816a3", status="md5_exact"),
|
||||
]
|
||||
kept = slots.format_decision(slots.arbitrate(conflict, "md5"))
|
||||
self.assertIn("a/neocdz.zip.ba20a7be, the file fbneo reads", kept)
|
||||
self.assertIn("b/neocdz.zip.712816a3, the file geolith reads", kept)
|
||||
self.assertEqual(conflict.files_read(), [
|
||||
("a/neocdz.zip.ba20a7be", ["fbneo"], "members_held"),
|
||||
("b/neocdz.zip.712816a3", ["geolith"], "md5_exact"),
|
||||
])
|
||||
|
||||
|
||||
class TheStrongestProofTakesAnUndeclaredPath(unittest.TestCase):
|
||||
"""No platform declares RetroDECK's hiscore.dat. fbalpha2012 asks for it
|
||||
|
||||
+169
-2
@@ -4,11 +4,13 @@
|
||||
under the target's name (Recalbox, Lakka, BizHawk, MiSTer, ROCKNIX). The
|
||||
conformance check read the same target without the --all tolerance, raised
|
||||
FileNotFoundError on the first of those packs and stopped the run in a
|
||||
traceback after every pack had been written.
|
||||
traceback after every pack had been written. Such a platform is now left
|
||||
out of a targeted --all run, and the check refuses it as the build does.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import contextlib
|
||||
import io
|
||||
@@ -31,7 +33,10 @@ class CheckReadsTheTargetLikeTheBuild(unittest.TestCase):
|
||||
contextlib.redirect_stdout(io.StringIO()):
|
||||
with self.assertRaises(ValueError):
|
||||
build_target_cores_cache(["recalbox"], "x86_64", platforms, is_all=True)
|
||||
self.assertIsNone(_target_cores_for("recalbox", "x86_64", platforms))
|
||||
# The build never packs such a platform under the target, so the
|
||||
# check refuses it too instead of reading the full pack.
|
||||
with self.assertRaises(FileNotFoundError):
|
||||
_target_cores_for("recalbox", "x86_64", platforms)
|
||||
|
||||
def test_a_platform_with_a_target_file_is_narrowed(self):
|
||||
cores = _target_cores_for("retroarch", "switch", str(REPO_ROOT / "platforms"))
|
||||
@@ -54,5 +59,167 @@ class ANameClaimsOnlyANarrowingThatHappened(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class AMistypedTargetIsRefusedFirst(unittest.TestCase):
|
||||
"""`--platform retroarch --target swich` took the dist lock and waited on
|
||||
the data directory refresh, 22 s here and hours behind a pipeline, before
|
||||
saying the target does not exist."""
|
||||
|
||||
def test_no_lock_and_no_refresh_before_the_refusal(self):
|
||||
from unittest import mock
|
||||
|
||||
import generate_pack
|
||||
|
||||
def touched(*_a, **_kw):
|
||||
raise AssertionError("lock or refresh taken before the target check")
|
||||
|
||||
argv = ["generate_pack.py", "--platform", "retroarch", "--target", "swich",
|
||||
"--output-dir", str(REPO_ROOT / "tmp" / "never-written")]
|
||||
err = io.StringIO()
|
||||
with mock.patch.object(sys, "argv", argv), \
|
||||
mock.patch.object(generate_pack, "_pack_output_lock", touched), \
|
||||
mock.patch.object(generate_pack, "hold_data_dirs", touched), \
|
||||
contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(err), \
|
||||
self.assertRaises(SystemExit) as ctx:
|
||||
generate_pack.main()
|
||||
self.assertEqual(ctx.exception.code, 1)
|
||||
self.assertIn("Unknown target 'swich'", err.getvalue())
|
||||
|
||||
|
||||
class ASystemTheTargetDropsIsNamedAsSuch(unittest.TestCase):
|
||||
"""`--platform retroarch --system sony-playstation-2 --target switch`:
|
||||
verify printed 0/0 with exit 0 under --strict, and generate_pack said
|
||||
RetroArch has no such system and suggested sony-playstation."""
|
||||
|
||||
PLATFORMS = str(REPO_ROOT / "platforms")
|
||||
|
||||
def test_both_tools_read_one_scope(self):
|
||||
from common import (
|
||||
load_emulator_profiles, load_platform_config, platform_scope_systems,
|
||||
)
|
||||
|
||||
config = load_platform_config("retroarch", self.PLATFORMS)
|
||||
profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
|
||||
switch = _target_cores_for("retroarch", "switch", self.PLATFORMS)
|
||||
wanted = ["sony-playstation-2"]
|
||||
self.assertEqual(platform_scope_systems(config, profiles, switch, wanted), {})
|
||||
self.assertEqual(list(platform_scope_systems(config, profiles, None, wanted)), wanted)
|
||||
|
||||
def test_the_message_names_the_target(self):
|
||||
from common import system_scope_error
|
||||
|
||||
config = {"systems": {"sony-playstation-2": {}, "sony-playstation": {}}}
|
||||
self.assertEqual(
|
||||
system_scope_error(config, "retroarch", ["sony-playstation-2"], "switch"),
|
||||
"no core retroarch runs on target switch emulates sony-playstation-2",
|
||||
)
|
||||
undeclared = system_scope_error(config, "retroarch", ["nec-pc-98"], "switch")
|
||||
self.assertTrue(undeclared.startswith("retroarch has no system matching nec-pc-98."))
|
||||
|
||||
def test_verify_refuses_it(self):
|
||||
import subprocess
|
||||
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "scripts/verify.py", "--platform", "retroarch",
|
||||
"--system", "sony-playstation-2", "--target", "switch", "--strict"],
|
||||
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=600, check=False,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 1, proc.stdout[-300:])
|
||||
self.assertIn("no core retroarch runs on target switch emulates sony-playstation-2",
|
||||
proc.stderr)
|
||||
|
||||
|
||||
class ASplitBuildsOnlyTheTargetsSystems(unittest.TestCase):
|
||||
"""`--split --target switch` asked for a part per declared system, and
|
||||
each one the target drops ended on 'retroarch has no system matching'."""
|
||||
|
||||
def test_no_part_is_asked_for_a_dropped_system(self):
|
||||
from unittest import mock
|
||||
|
||||
import generate_pack
|
||||
from common import load_emulator_profiles
|
||||
|
||||
platforms = str(REPO_ROOT / "platforms")
|
||||
profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
|
||||
asked: list[list[str]] = []
|
||||
|
||||
def record(*_a, **kw):
|
||||
if kw.get("system_filter"):
|
||||
asked.append(kw["system_filter"])
|
||||
return None
|
||||
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \
|
||||
mock.patch.object(generate_pack, "generate_pack", record), \
|
||||
mock.patch.object(generate_pack, "_collect_emulator_extras",
|
||||
return_value=[]):
|
||||
generate_pack.generate_split_packs(
|
||||
"retroarch", platforms, {"files": {}}, "bios", out,
|
||||
emu_profiles=profiles,
|
||||
target_cores=_target_cores_for("retroarch", "switch", platforms),
|
||||
target_name="nintendo-switch",
|
||||
)
|
||||
systems = {sid for ids in asked for sid in ids}
|
||||
self.assertTrue(systems)
|
||||
self.assertNotIn("sony-playstation-2", systems)
|
||||
self.assertIn("sony-playstation", systems)
|
||||
|
||||
|
||||
class APlatformWithoutTheSystemIsOutOfScope(unittest.TestCase):
|
||||
"""`--all --system sony-playstation-2` built the five PS2 packs and
|
||||
exited 1 on 'no pack for misterfpga', which has no PS2."""
|
||||
|
||||
def _scope(self, **args):
|
||||
import argparse
|
||||
|
||||
import generate_pack
|
||||
from common import load_emulator_profiles
|
||||
|
||||
values = {"platforms_dir": str(REPO_ROOT / "platforms"), "target": None,
|
||||
"all": True}
|
||||
values.update(args)
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
|
||||
kept = generate_pack._platforms_with_systems(
|
||||
argparse.Namespace(**values), ["retroarch", "misterfpga"],
|
||||
["sony-playstation-2"],
|
||||
load_emulator_profiles(str(REPO_ROOT / "emulators")), {},
|
||||
)
|
||||
return kept, out.getvalue()
|
||||
|
||||
def test_all_skips_it(self):
|
||||
kept, out = self._scope()
|
||||
self.assertEqual(kept, ["retroarch"])
|
||||
self.assertIn("INFO: Skipping misterfpga: no system sony-playstation-2", out)
|
||||
|
||||
def test_a_named_platform_is_refused(self):
|
||||
with self.assertRaises(SystemExit) as ctx:
|
||||
self._scope(all=False)
|
||||
self.assertEqual(ctx.exception.code, 1)
|
||||
|
||||
|
||||
class TheSplitDirectoryReadsThePackStem(unittest.TestCase):
|
||||
"""The split prefix was assembled by hand: a version 'release/1.22' made
|
||||
'RetroArch_release/1.22_Split', a nested directory the check never
|
||||
found, beside a full pack named 'RetroArch_release-1.22_BIOS_Pack.zip'."""
|
||||
|
||||
def test_a_reserved_character_is_made_safe_as_in_the_full_pack(self):
|
||||
from unittest import mock
|
||||
|
||||
import generate_pack
|
||||
|
||||
config = {"platform": "Foo: Bar", "version": "release/1.22", "systems": {}}
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \
|
||||
mock.patch.object(generate_pack, "load_platform_config",
|
||||
return_value=config), \
|
||||
mock.patch.object(generate_pack, "_collect_emulator_extras",
|
||||
return_value=[]):
|
||||
generate_pack.generate_split_packs(
|
||||
"foo", "platforms", {"files": {}}, "bios", out, emu_profiles={},
|
||||
)
|
||||
stem = generate_pack._platform_pack_stem(["foo"], "foo", "platforms")
|
||||
self.assertEqual(os.listdir(out), [f"{stem}_Split"])
|
||||
self.assertEqual(stem, "Foo-_Bar_release-1.22")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+116
-1
@@ -132,7 +132,7 @@ class TheVerifySummaryCountsDiscrepancies(unittest.TestCase):
|
||||
({"severity_counts": {}, "details": details}, ["RetroArch"]),
|
||||
({"severity_counts": {}, "details": []}, ["Batocera"]),
|
||||
])
|
||||
self.assertIn("1 discrepancy (RetroArch)", summary)
|
||||
self.assertIn("1 discrepancy (RetroArch 1)", summary)
|
||||
|
||||
def test_a_shared_destination_names_the_path_and_both_sides(self):
|
||||
result = {"name": "disk.rom", "status": verify.Status.UNTESTED}
|
||||
@@ -145,6 +145,121 @@ class TheVerifySummaryCountsDiscrepancies(unittest.TestCase):
|
||||
self.assertIn(part, message)
|
||||
|
||||
|
||||
class EveryPlatformInTheSummaryCarriesItsCount(unittest.TestCase):
|
||||
"""'17 discrepancies (Lakka 5, RetroArch 5, RetroBat 2, RetroDECK 4,
|
||||
ROCKNIX)': the one platform with a single entry had no number, and the
|
||||
named counts did not add up to the total."""
|
||||
|
||||
def test_a_count_of_one_is_written(self):
|
||||
one = {"severity_counts": {verify.Severity.WARNING: 1}, "details": []}
|
||||
two = {"severity_counts": {verify.Severity.WARNING: 2}, "details": []}
|
||||
summary = verify._overall_summary([(two, ["Lakka"]), (one, ["ROCKNIX"])])
|
||||
self.assertIn("3 warning (Lakka 2, ROCKNIX 1)", summary)
|
||||
|
||||
|
||||
class HeldButNotPackedNamesEachReason(unittest.TestCase):
|
||||
"""'Core files held but not packed: 16 (a packed file occupies its
|
||||
directory)' on Lakka, where 14 of the 16 are installed with their
|
||||
emulator: the first entry's reason stood for the whole count."""
|
||||
|
||||
@staticmethod
|
||||
def _entry(name: str, reason: str, category: str = "bios") -> dict:
|
||||
return {"name": name, "emulator": "x", "required": True, "in_repo": True,
|
||||
"in_pack": False, "omitted": reason, "category": category}
|
||||
|
||||
def _printed(self, entries: list[dict]) -> str:
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out):
|
||||
verify._print_undeclared_section({"undeclared_files": entries}, False)
|
||||
return out.getvalue()
|
||||
|
||||
def test_each_reason_carries_its_count(self):
|
||||
entries = [self._entry("a.rom", "a packed file occupies its directory")]
|
||||
entries += [self._entry(f"{i}.rom", "installed with its emulator") for i in range(3)]
|
||||
entries += [self._entry(f"g{i}", "installed with its emulator", "game_data")
|
||||
for i in range(2)]
|
||||
entries.append(self._entry("g9", "no platform slug for its system", "game_data"))
|
||||
text = self._printed(entries)
|
||||
self.assertIn(
|
||||
"Core files held but not packed: 4 (3 installed with its emulator, "
|
||||
"1 a packed file occupies its directory)", text)
|
||||
self.assertIn(
|
||||
"Game data held but not packed: 3 (2 installed with its emulator, "
|
||||
"1 no platform slug for its system)", text)
|
||||
|
||||
def test_a_missing_file_follows_the_line_that_counts_it(self):
|
||||
"""Indented under 'held but not packed: 102 (installed with its
|
||||
emulator)', Batocera's one missing sfiii2a.zip read as one of them."""
|
||||
missing = {"name": "sfiii2a.zip", "emulator": "FinalBurn Neo", "required": True,
|
||||
"in_repo": False, "category": "bios"}
|
||||
lines = self._printed(
|
||||
[missing, self._entry("a.rom", "installed with its emulator")]
|
||||
).splitlines()
|
||||
self.assertTrue(lines[0].startswith(" Core files: "), lines)
|
||||
self.assertIn("MISSING (required): FinalBurn Neo needs sfiii2a.zip", lines[1])
|
||||
self.assertTrue(lines[2].startswith(" Core files held but not packed: "), lines)
|
||||
|
||||
def test_one_name_read_at_two_paths_prints_both_paths(self):
|
||||
"""MyMinUI printed 'FinalBurn Neo needs sfiii2a.zip' twice, word for
|
||||
word, for Bios/FBN/ and Bios/NG/."""
|
||||
base = {"name": "sfiii2a.zip", "emulator": "FinalBurn Neo", "required": True,
|
||||
"in_repo": False, "category": "bios"}
|
||||
lone = dict(base, name="other.zip", path="Bios/FBN/other.zip")
|
||||
text = self._printed([dict(base, path="Bios/FBN/sfiii2a.zip"),
|
||||
dict(base, path="Bios/NG/sfiii2a.zip"), lone])
|
||||
self.assertIn("FinalBurn Neo needs Bios/FBN/sfiii2a.zip", text)
|
||||
self.assertIn("FinalBurn Neo needs Bios/NG/sfiii2a.zip", text)
|
||||
self.assertIn("FinalBurn Neo needs other.zip\n", text)
|
||||
|
||||
def test_a_single_reason_reads_as_before(self):
|
||||
text = self._printed([self._entry("a.rom", "installed with its emulator")] * 2)
|
||||
self.assertIn("Core files held but not packed: 2 (installed with its emulator)", text)
|
||||
|
||||
|
||||
class APlatformWithoutTargetsSaysSo(unittest.TestCase):
|
||||
"""`--platform misterfpga --target switch` printed 'No target config for
|
||||
platform 'misterfpga': platforms/targets/misterfpga.yml', which reads as
|
||||
a file missing from the repository."""
|
||||
|
||||
def test_the_message_names_the_reason_and_the_platforms_with_targets(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
(Path(tmp) / "targets").mkdir()
|
||||
for name in ("retroarch", "batocera", "_overrides"):
|
||||
(Path(tmp) / "targets" / f"{name}.yml").write_text("targets: {}\n")
|
||||
with self.assertRaises(FileNotFoundError) as caught:
|
||||
common.load_target_config("misterfpga", "switch", tmp)
|
||||
self.assertEqual(
|
||||
str(caught.exception),
|
||||
"misterfpga has no hardware targets: drop --target "
|
||||
"(platforms with targets: batocera, retroarch)",
|
||||
)
|
||||
|
||||
|
||||
class OneNameListsThePlatforms(unittest.TestCase):
|
||||
"""install.py --list-platforms, generate_pack.py --list, and no listing
|
||||
at all in verify.py: three scripts, three answers to one question."""
|
||||
|
||||
def _run(self, script: str, flag: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
[sys.executable, f"scripts/{script}", flag], capture_output=True, text=True,
|
||||
cwd=str(REPO_ROOT), timeout=120, check=False,
|
||||
)
|
||||
|
||||
def test_both_tools_take_it_and_agree(self):
|
||||
verify_out = self._run("verify.py", "--list-platforms")
|
||||
pack_out = self._run("generate_pack.py", "--list-platforms")
|
||||
self.assertEqual(verify_out.returncode, 0, verify_out.stderr)
|
||||
self.assertEqual(pack_out.returncode, 0, pack_out.stderr)
|
||||
self.assertEqual(verify_out.stdout, pack_out.stdout)
|
||||
self.assertIn("retroarch", verify_out.stdout.split())
|
||||
self.assertIn("--list-platforms", (REPO_ROOT / "install.py").read_text())
|
||||
|
||||
def test_the_installer_name_is_the_one_documented(self):
|
||||
tools = (REPO_ROOT / "wiki" / "tools.md").read_text()
|
||||
self.assertIn("generate_pack.py --list-platforms", tools)
|
||||
self.assertIn("verify.py --list-platforms", tools)
|
||||
|
||||
|
||||
class OneSpellingPerOption(unittest.TestCase):
|
||||
"""verify.py -p batocera worked and slots.py -p batocera was refused."""
|
||||
|
||||
|
||||
@@ -116,6 +116,24 @@ class VerifyPackChoice(unittest.TestCase):
|
||||
dict(zip((name for _p, name in checked), self.variants)), names
|
||||
)
|
||||
|
||||
def test_a_target_the_platform_lacks_is_refused(self):
|
||||
"""`--platform bizhawk --target rpi4typo --verify-packs` verified the
|
||||
full pack and said `all OK`, where the build refused the target."""
|
||||
import contextlib
|
||||
import io
|
||||
|
||||
bizhawk = self.gp._platform_pack_stem(["bizhawk"], "bizhawk", "platforms")
|
||||
self._touch(f"{bizhawk}_BIOS_Pack.zip")
|
||||
self._touch(f"{self.stem}_BIOS_Pack.zip")
|
||||
for platform, target in (("bizhawk", "rpi4typo"), ("retroarch", "swich")):
|
||||
with self.subTest(platform=platform), \
|
||||
contextlib.redirect_stdout(io.StringIO()), \
|
||||
contextlib.redirect_stderr(io.StringIO()), \
|
||||
self.assertRaises(SystemExit) as ctx:
|
||||
self._run(self._args(platform=platform, target=target))
|
||||
self.assertEqual(ctx.exception.code, 1)
|
||||
self.assertEqual(self.variants, [])
|
||||
|
||||
def test_the_run_ends_on_a_summary(self):
|
||||
"""A failing pack was named only above its own error lines, and the
|
||||
log ended on the last platform's OK line."""
|
||||
|
||||
@@ -89,6 +89,12 @@ class ReleaseAssetNotCached(unittest.TestCase):
|
||||
self.assertEqual(entry["status"], Status.OK)
|
||||
self.assertEqual(entry["release_asset"], REGISTERED[0])
|
||||
|
||||
def test_its_path_is_a_pack_destination(self):
|
||||
"""The pack holds the fetched file at its path: absent from
|
||||
pack_destinations, step 5 reported it 'pack only'."""
|
||||
report = verify_platform(self.config, self.db, "emulators", {})
|
||||
self.assertIn("big.bin", report["pack_destinations"])
|
||||
|
||||
def test_offline_counts_it_missing(self):
|
||||
packresolve.set_offline(True)
|
||||
self.assertEqual(self._entry()["status"], Status.MISSING)
|
||||
@@ -117,6 +123,59 @@ class ReleaseAssetNotCachedMd5(ReleaseAssetNotCached):
|
||||
mode = "md5"
|
||||
|
||||
|
||||
class CoreExtraReleaseAssetNotCached(ReleaseAssetNotCached):
|
||||
"""The same asset read by a core and declared by no platform entry: the
|
||||
builder fetched and packed it online, verify's replay of the extras
|
||||
resolved it offline and left its path out of the pack."""
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.config["systems"]["sys"]["files"] = []
|
||||
self.config["cores"] = ["bigcore"]
|
||||
Path("platforms/big.yml").write_text(yaml.dump(self.config))
|
||||
self.profiles = {"bigcore": {
|
||||
"emulator": "Big Core", "type": "libretro", "systems": ["sys"],
|
||||
"files": [{"name": "big.bin", "required": True, "sha1": SHA1}],
|
||||
}}
|
||||
Path("emulators/bigcore.yml").write_text(yaml.dump(self.profiles["bigcore"]))
|
||||
common._platform_config_cache.clear()
|
||||
common._emulator_profiles_cache.clear()
|
||||
|
||||
def _report(self) -> dict:
|
||||
return verify_platform(self.config, self.db, "emulators", self.profiles)
|
||||
|
||||
def test_counted_as_fetched_when_the_pack_is_built(self):
|
||||
report = self._report()
|
||||
self.assertIn("big.bin", report["pack_destinations"])
|
||||
self.assertEqual(report["packed_extras"]["bios"], 1)
|
||||
|
||||
def test_its_path_is_a_pack_destination(self):
|
||||
cache = Path("cache/big.bin")
|
||||
cache.parent.mkdir()
|
||||
cache.write_bytes(PAYLOAD)
|
||||
with mock.patch.object(
|
||||
packresolve, "fetch_large_file", return_value=str(cache)
|
||||
):
|
||||
zip_path = builder.generate_pack(
|
||||
"big", "platforms", self.db, "bios", "out",
|
||||
emulators_dir="emulators", emu_profiles=self.profiles, offline=False,
|
||||
)
|
||||
with zipfile.ZipFile(zip_path) as zf:
|
||||
packed = {n for n in zf.namelist() if not n.endswith("/")}
|
||||
self.assertIn("big.bin", packed)
|
||||
self.assertIn("big.bin", self._report()["pack_destinations"])
|
||||
|
||||
def test_offline_counts_it_missing(self):
|
||||
packresolve.set_offline(True)
|
||||
self.assertNotIn("big.bin", self._report()["pack_destinations"])
|
||||
|
||||
def test_an_asset_with_other_content_is_not_counted(self):
|
||||
self.profiles["bigcore"]["files"][0]["sha1"] = "0" * 40
|
||||
self.assertNotIn("big.bin", self._report()["pack_destinations"])
|
||||
|
||||
test_the_online_build_packs_it = None
|
||||
|
||||
|
||||
class OfflinePipelineVerifiesOffline(unittest.TestCase):
|
||||
"""The offline pipeline builds packs without release assets: its verify
|
||||
step has to count them the same way, or step 5 compares two answers."""
|
||||
|
||||
@@ -117,7 +117,8 @@ RetroDECK: 2007/2008 OK, 1 untested [md5]
|
||||
|
||||
The run prints more `DISCREPANCY` lines than this excerpt shows. The core
|
||||
and game data lines count what the emulator cores add beyond the platform's
|
||||
own list. A file held but not packed is left out for the reason in parentheses.
|
||||
own list. A file held but not packed is left out for the reason in parentheses, each
|
||||
reason with its count when the files have several.
|
||||
|
||||
`--verbose` also lists the passing entries that an emulator validates, with
|
||||
the check and the source lines behind it:
|
||||
|
||||
+2
-1
@@ -134,6 +134,7 @@ python scripts/verify.py --platform recalbox --region us # regional priority l
|
||||
python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named
|
||||
python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds
|
||||
python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build
|
||||
python scripts/verify.py --list-platforms # list all platforms
|
||||
python scripts/verify.py --list-emulators # list all emulators
|
||||
python scripts/verify.py --list-systems # list all systems
|
||||
python scripts/verify.py --platform retroarch --list-targets # list available targets
|
||||
@@ -208,7 +209,7 @@ python scripts/generate_pack.py --all --all-variants --output-dir dist/ # all 6
|
||||
python scripts/generate_pack.py --all --all-variants --verify-packs --output-dir dist/
|
||||
|
||||
# Listing and integrity
|
||||
python scripts/generate_pack.py --list # list available platforms
|
||||
python scripts/generate_pack.py --list-platforms # list available platforms
|
||||
python scripts/generate_pack.py --list-emulators
|
||||
python scripts/generate_pack.py --list-systems
|
||||
python scripts/generate_pack.py --platform retroarch --list-targets
|
||||
|
||||
Reference in new issue
Block a user