mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-06 01:53:56 -06:00
chore: merge audit batch, verify and pack
This commit is contained in:
commit
3f50163b0a
15 files changed
+748
-127
No files matched your search
+64
-1
@@ -664,8 +664,15 @@ def load_target_config(
|
|||||||
targets_dir = os.path.join(platforms_dir, "targets")
|
targets_dir = os.path.join(platforms_dir, "targets")
|
||||||
target_file = os.path.join(targets_dir, f"{platform_name}.yml")
|
target_file = os.path.join(targets_dir, f"{platform_name}.yml")
|
||||||
if not os.path.exists(target_file):
|
if not os.path.exists(target_file):
|
||||||
|
# The expected state for a platform that ships one build, said as
|
||||||
|
# such: the file path read like a file missing from the repository.
|
||||||
|
with_targets = sorted(
|
||||||
|
Path(f).stem for f in os.listdir(targets_dir)
|
||||||
|
if f.endswith(".yml") and not f.startswith("_")
|
||||||
|
)
|
||||||
raise FileNotFoundError(
|
raise FileNotFoundError(
|
||||||
f"No target config for platform '{platform_name}': {target_file}"
|
f"{platform_name} has no hardware targets: drop --target "
|
||||||
|
f"(platforms with targets: {', '.join(with_targets)})"
|
||||||
)
|
)
|
||||||
with open(target_file) as f:
|
with open(target_file) as f:
|
||||||
data = yaml_load(f) or {}
|
data = yaml_load(f) or {}
|
||||||
@@ -2986,6 +2993,46 @@ def filter_systems_by_target(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def platform_scope_systems(
|
||||||
|
config: dict,
|
||||||
|
profiles: dict[str, dict],
|
||||||
|
target_cores: set[str] | None,
|
||||||
|
system_filter: list[str] | None = None,
|
||||||
|
) -> dict[str, dict]:
|
||||||
|
"""The systems a pack or a report of a platform covers.
|
||||||
|
|
||||||
|
Those a core of the target runs, then the ones a --system list keeps.
|
||||||
|
verify.py and generate_pack.py read this one function.
|
||||||
|
"""
|
||||||
|
plat_cores = resolve_platform_cores(config, profiles) if target_cores else None
|
||||||
|
systems = filter_systems_by_target(
|
||||||
|
config.get("systems", {}), profiles, target_cores, platform_cores=plat_cores
|
||||||
|
)
|
||||||
|
return select_systems(systems, system_filter) if system_filter else systems
|
||||||
|
|
||||||
|
|
||||||
|
def system_scope_error(
|
||||||
|
config: dict, platform_name: str, system_filter: list[str], target: str | None
|
||||||
|
) -> str:
|
||||||
|
"""Why a --system list leaves a platform no system.
|
||||||
|
|
||||||
|
A system the platform declares but no core of the target runs was
|
||||||
|
reported as a system the platform lacks, with a lookalike to try and a
|
||||||
|
listing that refuses --target.
|
||||||
|
"""
|
||||||
|
systems = config.get("systems", {})
|
||||||
|
declared = select_systems(systems, system_filter)
|
||||||
|
if declared and target:
|
||||||
|
return (
|
||||||
|
f"no core {platform_name} runs on target {target} emulates "
|
||||||
|
f"{', '.join(sorted(declared))}"
|
||||||
|
)
|
||||||
|
hint = suggest_names(
|
||||||
|
system_filter, sorted(systems), f"--platform {platform_name} --list-systems"
|
||||||
|
)
|
||||||
|
return f"{platform_name} has no system matching {', '.join(system_filter)}. {hint}"
|
||||||
|
|
||||||
|
|
||||||
def platform_declarations(
|
def platform_declarations(
|
||||||
config: dict, db: dict, *, enrich: bool = True
|
config: dict, db: dict, *, enrich: bool = True
|
||||||
) -> dict[str, list[tuple[int | None, str]]]:
|
) -> dict[str, list[tuple[int | None, str]]]:
|
||||||
@@ -3145,6 +3192,22 @@ def list_system_ids(emulators_dir: str) -> None:
|
|||||||
print(f" {spelling:35s} ({count} emulator{'s' if count > 1 else ''})")
|
print(f" {spelling:35s} ({count} emulator{'s' if count > 1 else ''})")
|
||||||
|
|
||||||
|
|
||||||
|
def print_platform_listing(platforms_dir: str) -> None:
|
||||||
|
"""Every registered platform, one per line, marked when archived or
|
||||||
|
served by the installer only.
|
||||||
|
|
||||||
|
Every platform takes --platform. --all covers the active ones, and the
|
||||||
|
pack builder's --all the release packs. Printed by --list-platforms in
|
||||||
|
verify.py and generate_pack.py, the name install.py gives it.
|
||||||
|
"""
|
||||||
|
active = set(list_registered_platforms(platforms_dir))
|
||||||
|
released = set(list_release_pack_platforms(platforms_dir, include_archived=True))
|
||||||
|
for name in list_registered_platforms(platforms_dir, include_archived=True):
|
||||||
|
marks = [m for m, on in (("archived", name not in active),
|
||||||
|
("installer only", name not in released)) if on]
|
||||||
|
print(f"{name} ({', '.join(marks)})" if marks else name)
|
||||||
|
|
||||||
|
|
||||||
def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None:
|
def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None:
|
||||||
"""Print system IDs from a platform's YAML config, with the platform's
|
"""Print system IDs from a platform's YAML config, with the platform's
|
||||||
own names for them, which --system also accepts."""
|
own names for them, which --system also accepts."""
|
||||||
|
|||||||
+107
-72
@@ -53,7 +53,10 @@ from common import (
|
|||||||
list_platform_system_ids,
|
list_platform_system_ids,
|
||||||
platform_system_names,
|
platform_system_names,
|
||||||
resolve_system_arg,
|
resolve_system_arg,
|
||||||
|
print_platform_listing,
|
||||||
select_systems,
|
select_systems,
|
||||||
|
platform_scope_systems,
|
||||||
|
system_scope_error,
|
||||||
canonical_target_name,
|
canonical_target_name,
|
||||||
platform_files_target,
|
platform_files_target,
|
||||||
list_registered_platforms,
|
list_registered_platforms,
|
||||||
@@ -753,31 +756,13 @@ def generate_pack(
|
|||||||
target_cores=target_cores, source=source,
|
target_cores=target_cores, source=source,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Filter systems by target if specified
|
pack_systems = platform_scope_systems(
|
||||||
plat_cores = (
|
config, emu_profiles or {}, target_cores, system_filter
|
||||||
resolve_platform_cores(config, emu_profiles or {}) if target_cores else None
|
|
||||||
)
|
)
|
||||||
pack_systems = filter_systems_by_target(
|
if system_filter and not pack_systems:
|
||||||
config.get("systems", {}),
|
reason = system_scope_error(config, platform_name, system_filter, target_name)
|
||||||
emu_profiles or {},
|
print(f" WARNING: {reason}")
|
||||||
target_cores,
|
return None
|
||||||
platform_cores=plat_cores,
|
|
||||||
)
|
|
||||||
|
|
||||||
if system_filter:
|
|
||||||
filtered = select_systems(pack_systems, system_filter)
|
|
||||||
if not filtered:
|
|
||||||
hint = suggest_names(
|
|
||||||
system_filter,
|
|
||||||
sorted(pack_systems),
|
|
||||||
f"--platform {platform_name} --list-systems",
|
|
||||||
)
|
|
||||||
print(
|
|
||||||
f" WARNING: {platform_name} has no system matching "
|
|
||||||
f"{', '.join(system_filter)}. {hint}"
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
pack_systems = filtered
|
|
||||||
if extras_only:
|
if extras_only:
|
||||||
# The part of a split that carries the core extras no platform system
|
# The part of a split that carries the core extras no platform system
|
||||||
# owns: the platform's own files and trees went to the other parts.
|
# owns: the platform's own files and trees went to the other parts.
|
||||||
@@ -1882,11 +1867,6 @@ def generate_system_pack(
|
|||||||
shutil.rmtree(scratch, ignore_errors=True)
|
shutil.rmtree(scratch, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
def list_platforms(platforms_dir: str) -> list[str]:
|
|
||||||
"""List available platform names from registry."""
|
|
||||||
return list_registered_platforms(platforms_dir, include_archived=True)
|
|
||||||
|
|
||||||
|
|
||||||
def _system_display_name(system_id: str) -> str:
|
def _system_display_name(system_id: str) -> str:
|
||||||
"""Convert system ID to display name for ZIP naming."""
|
"""Convert system ID to display name for ZIP naming."""
|
||||||
s = system_id.lower().replace("_", "-")
|
s = system_id.lower().replace("_", "-")
|
||||||
@@ -1935,7 +1915,6 @@ def generate_split_packs(
|
|||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
"""Generate split packs (one ZIP per system or manufacturer)."""
|
"""Generate split packs (one ZIP per system or manufacturer)."""
|
||||||
config = load_platform_config(platform_name, platforms_dir)
|
config = load_platform_config(platform_name, platforms_dir)
|
||||||
platform_display = config.get("platform", platform_name)
|
|
||||||
# The split directory carries the same tags as the packs inside it, or two
|
# The split directory carries the same tags as the packs inside it, or two
|
||||||
# differently narrowed runs write into one another.
|
# differently narrowed runs write into one another.
|
||||||
split_tags = "".join(
|
split_tags = "".join(
|
||||||
@@ -1951,13 +1930,19 @@ def generate_split_packs(
|
|||||||
# The parts carry the version, so the directory does too: a rescrape that
|
# The parts carry the version, so the directory does too: a rescrape that
|
||||||
# moves the version would otherwise leave the old parts beside the new
|
# moves the version would otherwise leave the old parts beside the new
|
||||||
# under one SHA256SUMS.txt.
|
# under one SHA256SUMS.txt.
|
||||||
version = config.get("version", config.get("dat_version", ""))
|
# Named by the stem the full pack reads: assembled here by hand, a '/' in
|
||||||
ver_tag = f"_{version.replace(' ', '')}" if version else ""
|
# a version nested the directory where the check never looked for it.
|
||||||
part_prefix = f"{platform_display.replace(' ', '_')}{ver_tag}{split_tags}"
|
part_prefix = (
|
||||||
|
f"{_platform_pack_stem([platform_name], platform_name, platforms_dir)}{split_tags}"
|
||||||
|
)
|
||||||
split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split")
|
split_dir = os.path.join(output_dir, f"{part_prefix}{group_tag}_Split")
|
||||||
os.makedirs(split_dir, exist_ok=True)
|
os.makedirs(split_dir, exist_ok=True)
|
||||||
|
|
||||||
systems = config.get("systems", {})
|
if emu_profiles is None:
|
||||||
|
emu_profiles = load_emulator_profiles(emulators_dir)
|
||||||
|
# The systems a core of the target runs: a part per declared system sent
|
||||||
|
# each one the target drops to a refusal naming it as unknown.
|
||||||
|
systems = platform_scope_systems(config, emu_profiles, target_cores)
|
||||||
|
|
||||||
if group_by == "manufacturer":
|
if group_by == "manufacturer":
|
||||||
groups = _group_systems_by_manufacturer(systems, db, bios_dir)
|
groups = _group_systems_by_manufacturer(systems, db, bios_dir)
|
||||||
@@ -1966,8 +1951,6 @@ def generate_split_packs(
|
|||||||
|
|
||||||
# Pre-compute core extras once (expensive: scans 260+ emulator profiles)
|
# Pre-compute core extras once (expensive: scans 260+ emulator profiles)
|
||||||
# then distribute per group based on emulator system overlap
|
# then distribute per group based on emulator system overlap
|
||||||
if emu_profiles is None:
|
|
||||||
emu_profiles = load_emulator_profiles(emulators_dir)
|
|
||||||
base_dest = config.get("base_destination", "")
|
base_dest = config.get("base_destination", "")
|
||||||
# Each part cuts the platform's own entries from the full profiles. The
|
# Each part cuts the platform's own entries from the full profiles. The
|
||||||
# extras are collected once here, already without the cores' game data.
|
# extras are collected once here, already without the cores' game data.
|
||||||
@@ -2373,6 +2356,46 @@ def _resolve_systems(args: argparse.Namespace, parser: argparse.ArgumentParser)
|
|||||||
parser.error(f"--system {args.system}: {exc}")
|
parser.error(f"--system {args.system}: {exc}")
|
||||||
|
|
||||||
|
|
||||||
|
def _platforms_with_systems(
|
||||||
|
args: argparse.Namespace,
|
||||||
|
platforms: list[str],
|
||||||
|
system_filter: list[str],
|
||||||
|
emu_profiles: dict,
|
||||||
|
target_cores_cache: dict[str, set[str] | None],
|
||||||
|
) -> list[str]:
|
||||||
|
"""The platforms a --system list leaves a system to pack, before any lock.
|
||||||
|
|
||||||
|
Under --all a platform without one is out of scope, as a platform
|
||||||
|
without the target is: `--all --system sony-playstation-2` built every
|
||||||
|
PS2 pack and exited 1 on MiSTer, which has no PS2. Under --platform the
|
||||||
|
request is refused here, with its reason.
|
||||||
|
"""
|
||||||
|
kept = []
|
||||||
|
for name in platforms:
|
||||||
|
config = load_platform_config(name, args.platforms_dir)
|
||||||
|
tc = target_cores_cache.get(name) if args.target else None
|
||||||
|
if platform_scope_systems(config, emu_profiles, tc, system_filter):
|
||||||
|
kept.append(name)
|
||||||
|
continue
|
||||||
|
if not args.all:
|
||||||
|
reason = system_scope_error(config, name, system_filter, args.target)
|
||||||
|
print(f"ERROR: {reason}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
declared = select_systems(config.get("systems", {}), system_filter)
|
||||||
|
reason = (
|
||||||
|
system_scope_error(config, name, system_filter, args.target)
|
||||||
|
if declared and args.target
|
||||||
|
else f"no system {', '.join(system_filter)}"
|
||||||
|
)
|
||||||
|
print(f"INFO: Skipping {name}: {reason}")
|
||||||
|
if not kept:
|
||||||
|
where = f" on target {args.target}" if args.target else ""
|
||||||
|
print(f"ERROR: no platform packed here runs {', '.join(system_filter)}{where}",
|
||||||
|
file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return kept
|
||||||
|
|
||||||
|
|
||||||
def _refuse_for_verify_packs(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
def _refuse_for_verify_packs(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
||||||
"""What --verify-packs cannot be combined with, refused before any lock.
|
"""What --verify-packs cannot be combined with, refused before any lock.
|
||||||
|
|
||||||
@@ -2652,6 +2675,19 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
|
|||||||
else:
|
else:
|
||||||
raise ValueError("--verify-packs requires --platform or --all")
|
raise ValueError("--verify-packs requires --platform or --all")
|
||||||
|
|
||||||
|
# The target is checked as the build checks it: a mistyped one, or one
|
||||||
|
# the platform does not publish, was dropped and the full pack verified
|
||||||
|
# under it, `all OK` for a pack no build had made.
|
||||||
|
target_cores_cache: dict[str, set[str] | None] = {}
|
||||||
|
if args.target:
|
||||||
|
try:
|
||||||
|
target_cores_cache, platforms = build_target_cores_cache(
|
||||||
|
platforms, args.target, args.platforms_dir, is_all=args.all
|
||||||
|
)
|
||||||
|
except (FileNotFoundError, ValueError) as e:
|
||||||
|
print(f"ERROR: {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
all_ok = True
|
all_ok = True
|
||||||
verify_regions = getattr(args, "regions", None)
|
verify_regions = getattr(args, "regions", None)
|
||||||
verify_profiles = load_emulator_profiles(args.emulators_dir)
|
verify_profiles = load_emulator_profiles(args.emulators_dir)
|
||||||
@@ -2689,7 +2725,7 @@ def _run_verify_packs(args: argparse.Namespace) -> None:
|
|||||||
else:
|
else:
|
||||||
print(f" {platform_name}: SKIP (no pack in {args.output_dir})")
|
print(f" {platform_name}: SKIP (no pack in {args.output_dir})")
|
||||||
continue
|
continue
|
||||||
tc = _target_cores_for(platform_name, args.target, args.platforms_dir)
|
tc = target_cores_cache.get(platform_name)
|
||||||
# A group pack and a pack built for the platform alone can both be
|
# A group pack and a pack built for the platform alone can both be
|
||||||
# present, and each one is a pack of this platform.
|
# present, and each one is a pack of this platform.
|
||||||
for pack_name in found:
|
for pack_name in found:
|
||||||
@@ -3057,7 +3093,7 @@ def _refuse_unapplied_flags(args: argparse.Namespace, parser: argparse.ArgumentP
|
|||||||
name for name, on in (
|
name for name, on in (
|
||||||
("--manifest-targets", args.manifest_targets),
|
("--manifest-targets", args.manifest_targets),
|
||||||
("--verify-packs", args.verify_packs and not args.all_variants),
|
("--verify-packs", args.verify_packs and not args.all_variants),
|
||||||
("--list", args.list),
|
("--list-platforms", args.list),
|
||||||
("--list-emulators", args.list_emulators),
|
("--list-emulators", args.list_emulators),
|
||||||
("--list-systems", args.list_systems),
|
("--list-systems", args.list_systems),
|
||||||
("--list-targets", args.list_targets),
|
("--list-targets", args.list_targets),
|
||||||
@@ -3083,7 +3119,7 @@ QUICK_MODE_FLAGS: dict[str, set[str]] = {
|
|||||||
},
|
},
|
||||||
# One manifest per hardware target, from the target files themselves.
|
# One manifest per hardware target, from the target files themselves.
|
||||||
"--manifest-targets": set(),
|
"--manifest-targets": set(),
|
||||||
"--list": set(),
|
"--list-platforms": set(),
|
||||||
"--list-emulators": set(),
|
"--list-emulators": set(),
|
||||||
"--list-systems": {"--platform"},
|
"--list-systems": {"--platform"},
|
||||||
"--list-targets": {"--platform"},
|
"--list-targets": {"--platform"},
|
||||||
@@ -3112,7 +3148,7 @@ def _given_flags(args: argparse.Namespace) -> set[str]:
|
|||||||
"--target": args.target,
|
"--target": args.target,
|
||||||
"--region": args.region,
|
"--region": args.region,
|
||||||
"--one-per-slot": args.one_per_slot,
|
"--one-per-slot": args.one_per_slot,
|
||||||
"--list": args.list,
|
"--list-platforms": args.list,
|
||||||
"--list-emulators": args.list_emulators,
|
"--list-emulators": args.list_emulators,
|
||||||
"--list-systems": args.list_systems,
|
"--list-systems": args.list_systems,
|
||||||
"--list-targets": args.list_targets,
|
"--list-targets": args.list_targets,
|
||||||
@@ -3209,7 +3245,11 @@ def _main(held: contextlib.ExitStack) -> None:
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Pack the data directories as cached, still fetching release assets",
|
help="Pack the data directories as cached, still fetching release assets",
|
||||||
)
|
)
|
||||||
parser.add_argument("--list", action="store_true", help="List available platforms")
|
# install.py's name. --list stays for the scripts that already type it.
|
||||||
|
parser.add_argument(
|
||||||
|
"--list-platforms", "--list", dest="list", action="store_true",
|
||||||
|
help="List available platforms",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--required-only",
|
"--required-only",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -3311,14 +3351,7 @@ def _main(held: contextlib.ExitStack) -> None:
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
if args.list:
|
if args.list:
|
||||||
# Every platform builds with --platform. --all builds the release
|
print_platform_listing(args.platforms_dir)
|
||||||
# packs only, the others are served by the installer.
|
|
||||||
active = set(list_registered_platforms(args.platforms_dir))
|
|
||||||
released = set(list_release_pack_platforms(args.platforms_dir, include_archived=True))
|
|
||||||
for p in list_platforms(args.platforms_dir):
|
|
||||||
marks = [m for m, on in (("archived", p not in active),
|
|
||||||
("installer only", p not in released)) if on]
|
|
||||||
print(f"{p} ({', '.join(marks)})" if marks else p)
|
|
||||||
return
|
return
|
||||||
if args.list_emulators:
|
if args.list_emulators:
|
||||||
list_emulator_profiles(args.emulators_dir)
|
list_emulator_profiles(args.emulators_dir)
|
||||||
@@ -3464,6 +3497,28 @@ def _main(held: contextlib.ExitStack) -> None:
|
|||||||
parser.error("Specify --platform or --all")
|
parser.error("Specify --platform or --all")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# A mistyped target is refused before the output lock and the data
|
||||||
|
# directory refresh: checked after them, `--target swich` waited on a
|
||||||
|
# pipeline's caches before saying the target does not exist.
|
||||||
|
target_cores_cache: dict[str, set[str] | None] = {}
|
||||||
|
if args.target:
|
||||||
|
try:
|
||||||
|
target_cores_cache, platforms = build_target_cores_cache(
|
||||||
|
platforms,
|
||||||
|
args.target,
|
||||||
|
args.platforms_dir,
|
||||||
|
is_all=args.all,
|
||||||
|
)
|
||||||
|
except (FileNotFoundError, ValueError) as e:
|
||||||
|
print(f"ERROR: {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
emu_profiles = load_emulator_profiles(args.emulators_dir)
|
||||||
|
if system_filter:
|
||||||
|
platforms = _platforms_with_systems(
|
||||||
|
args, platforms, system_filter, emu_profiles, target_cores_cache
|
||||||
|
)
|
||||||
|
|
||||||
data_registry = load_data_dir_registry(args.platforms_dir)
|
data_registry = load_data_dir_registry(args.platforms_dir)
|
||||||
# The output directory is claimed before the data directory refresh: a
|
# The output directory is claimed before the data directory refresh: a
|
||||||
# refresh waits for every run that holds a cache, so a second build
|
# refresh waits for every run that holds a cache, so a second build
|
||||||
@@ -3483,21 +3538,6 @@ def _main(held: contextlib.ExitStack) -> None:
|
|||||||
if data_registry:
|
if data_registry:
|
||||||
hold_data_dirs(held, data_registry)
|
hold_data_dirs(held, data_registry)
|
||||||
|
|
||||||
emu_profiles = load_emulator_profiles(args.emulators_dir)
|
|
||||||
|
|
||||||
target_cores_cache: dict[str, set[str] | None] = {}
|
|
||||||
if args.target:
|
|
||||||
try:
|
|
||||||
target_cores_cache, platforms = build_target_cores_cache(
|
|
||||||
platforms,
|
|
||||||
args.target,
|
|
||||||
args.platforms_dir,
|
|
||||||
is_all=args.all,
|
|
||||||
)
|
|
||||||
except (FileNotFoundError, ValueError) as e:
|
|
||||||
print(f"ERROR: {e}", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
groups = group_identical_platforms(
|
groups = group_identical_platforms(
|
||||||
platforms, args.platforms_dir, target_cores_cache if args.target else None
|
platforms, args.platforms_dir, target_cores_cache if args.target else None
|
||||||
)
|
)
|
||||||
@@ -4877,17 +4917,12 @@ def _target_cores_for(
|
|||||||
) -> set[str] | None:
|
) -> set[str] | None:
|
||||||
"""Cores a target leaves available, or None when no target was asked for.
|
"""Cores a target leaves available, or None when no target was asked for.
|
||||||
|
|
||||||
None means no narrowing, which is what an unfiltered pack expects. Read
|
Called on a platform the build already packed under the target, so the
|
||||||
the way --all builds: a platform without a target file was packed
|
target is known to it: a platform without one is left out of the run.
|
||||||
unnarrowed under the target's name, and checking that pack must not
|
|
||||||
raise where building it did not.
|
|
||||||
"""
|
"""
|
||||||
if not target_name:
|
if not target_name:
|
||||||
return None
|
return None
|
||||||
try:
|
return load_target_config(platform_name, target_name, platforms_dir)
|
||||||
return load_target_config(platform_name, target_name, platforms_dir)
|
|
||||||
except (FileNotFoundError, ValueError):
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _expected_pack_names(
|
def _expected_pack_names(
|
||||||
|
|||||||
+14
-1
@@ -244,6 +244,7 @@ class ExtraResolver:
|
|||||||
offline: bool | None = True,
|
offline: bool | None = True,
|
||||||
*,
|
*,
|
||||||
profiles: dict,
|
profiles: dict,
|
||||||
|
pending_assets: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.profiles = profiles
|
self.profiles = profiles
|
||||||
self.db = db
|
self.db = db
|
||||||
@@ -253,10 +254,13 @@ class ExtraResolver:
|
|||||||
self.validation_index = validation_index
|
self.validation_index = validation_index
|
||||||
self.validation_owners = validation_owners
|
self.validation_owners = validation_owners
|
||||||
self.offline = offline
|
self.offline = offline
|
||||||
|
# verify's replay: never downloads, and counts the release asset an
|
||||||
|
# online build fetches, as it does for a platform entry.
|
||||||
|
self.pending_assets = pending_assets
|
||||||
|
|
||||||
def resolve(self, fe: dict, dest: str) -> tuple[str | None, str, str]:
|
def resolve(self, fe: dict, dest: str) -> tuple[str | None, str, str]:
|
||||||
"""(local copy or None, resolver status, the core's disagreement)."""
|
"""(local copy or None, resolver status, the core's disagreement)."""
|
||||||
from packresolve import resolve_file
|
from packresolve import pending_release_asset, resolve_file, run_is_offline
|
||||||
from validation import core_size_refusal, extra_alternatives, validated_choice
|
from validation import core_size_refusal, extra_alternatives, validated_choice
|
||||||
|
|
||||||
local_path, status = resolve_file(
|
local_path, status = resolve_file(
|
||||||
@@ -264,6 +268,15 @@ class ExtraResolver:
|
|||||||
dest_hint=fe.get("destination", ""),
|
dest_hint=fe.get("destination", ""),
|
||||||
data_dir_registry=self.data_registry, offline=self.offline,
|
data_dir_registry=self.data_registry, offline=self.offline,
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
self.pending_assets and status in ("not_found", "hash_mismatch")
|
||||||
|
and not run_is_offline()
|
||||||
|
):
|
||||||
|
# The asset the online builder fetches in resolve_file. Its record
|
||||||
|
# answers the declaration, and there are no bytes on disk to check.
|
||||||
|
asset = pending_release_asset(fe, self.db)
|
||||||
|
if asset:
|
||||||
|
return asset, "release_asset", ""
|
||||||
if status in ("not_found", "external", "user_provided") or not local_path:
|
if status in ("not_found", "external", "user_provided") or not local_path:
|
||||||
return None, status, ""
|
return None, status, ""
|
||||||
local_path, disagreement = validated_choice(
|
local_path, disagreement = validated_choice(
|
||||||
|
|||||||
+24
-9
@@ -30,7 +30,7 @@ from packextras import platform_region_groups
|
|||||||
from packpaths import _flat
|
from packpaths import _flat
|
||||||
from nativemode import reads_file_contents
|
from nativemode import reads_file_contents
|
||||||
import region as region_mod
|
import region as region_mod
|
||||||
from packresolve import resolve_file
|
from packresolve import resolve_declaration, resolve_file
|
||||||
from exporter.candidates import BIOS, EntryCategories
|
from exporter.candidates import BIOS, EntryCategories
|
||||||
from gamedata import without_game_data
|
from gamedata import without_game_data
|
||||||
from common import resolve_platform_cores
|
from common import resolve_platform_cores
|
||||||
@@ -544,6 +544,21 @@ def _structural_errors(zf: zipfile.ZipFile, zip_set: set) -> list[str]:
|
|||||||
return errors
|
return errors
|
||||||
|
|
||||||
|
|
||||||
|
def _shipped_copy(
|
||||||
|
entry: dict, db: dict, data_registry: dict | None, mode: str, hash_rule: str,
|
||||||
|
) -> str | None:
|
||||||
|
"""The file the builder packs for a declaration, or None.
|
||||||
|
|
||||||
|
The verdict verify applies before it lets a declaration hold its path.
|
||||||
|
"""
|
||||||
|
local, status = resolve_declaration(
|
||||||
|
entry, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True,
|
||||||
|
)
|
||||||
|
if local and baseline_verdict(entry, local, status, mode, hash_rule)[1] is None:
|
||||||
|
return local
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _builder_extras(
|
def _builder_extras(
|
||||||
config: dict,
|
config: dict,
|
||||||
emulators_dir: str,
|
emulators_dir: str,
|
||||||
@@ -563,15 +578,17 @@ def _builder_extras(
|
|||||||
"""{full destination: local copy} of each core extra the builder writes.
|
"""{full destination: local copy} of each core extra the builder writes.
|
||||||
|
|
||||||
The platform's paths come first, as in the builder: a destination the
|
The platform's paths come first, as in the builder: a destination the
|
||||||
pack holds whose declarations resolve to a file, or that the platform
|
pack holds whose declarations resolve to a file it ships, or that the
|
||||||
keeps for a user_provided or external entry. One whose declarations
|
platform keeps for a user_provided or external entry. One whose
|
||||||
resolve to nothing is left free, since the builder then places a core's
|
declarations resolve to nothing, or to a file the frontend's hash
|
||||||
file there.
|
refuses, is left free, since the builder then places a core's file
|
||||||
|
there: taken by the refused file, that core file was never compared.
|
||||||
"""
|
"""
|
||||||
from packextras import ExtraResolver, hidden_extras
|
from packextras import ExtraResolver, hidden_extras
|
||||||
from packresolve import resolve_declaration
|
|
||||||
from validation import _build_validation_index, destination_owners
|
from validation import _build_validation_index, destination_owners
|
||||||
|
|
||||||
|
mode = config.get("verification_mode", "existence")
|
||||||
|
hash_rule = config.get("hash_rule", "")
|
||||||
shipped: dict[str, str | None] = {}
|
shipped: dict[str, str | None] = {}
|
||||||
for expected, declarations in baseline_groups.items():
|
for expected, declarations in baseline_groups.items():
|
||||||
if expected not in zip_set and expected.lower() not in zip_lower:
|
if expected not in zip_set and expected.lower() not in zip_lower:
|
||||||
@@ -587,9 +604,7 @@ def _builder_extras(
|
|||||||
local = next(
|
local = next(
|
||||||
(
|
(
|
||||||
found for fe in declarations
|
found for fe in declarations
|
||||||
if (found := resolve_declaration(
|
if (found := _shipped_copy(fe, db, data_registry, mode, hash_rule))
|
||||||
fe, db, DEFAULT_BIOS_DIR, None, data_registry, None, offline=True,
|
|
||||||
)[0])
|
|
||||||
),
|
),
|
||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
|
|||||||
+23
-5
@@ -100,6 +100,20 @@ class Conflict:
|
|||||||
def emulators(self) -> list[str]:
|
def emulators(self) -> list[str]:
|
||||||
return sorted({c.emulator for c in self.profile_claims if c.emulator})
|
return sorted({c.emulator for c in self.profile_claims if c.emulator})
|
||||||
|
|
||||||
|
def files_read(self) -> list[tuple[str | None, list[str], str]]:
|
||||||
|
"""Each file the profiles want here, with the emulators reading it
|
||||||
|
and the proof of the first claim on it.
|
||||||
|
|
||||||
|
One file stood for every claim: Recalbox's neocdz.zip line gave
|
||||||
|
geolith the archive fbneo reads, not the one geolith pins.
|
||||||
|
"""
|
||||||
|
files: dict[str | None, tuple[list[str], str]] = {}
|
||||||
|
for claim in self.profile_claims:
|
||||||
|
emulators, status = files.setdefault(claim.local_path, ([], claim.status))
|
||||||
|
if claim.emulator and claim.emulator not in emulators:
|
||||||
|
emulators.append(claim.emulator)
|
||||||
|
return [(path, sorted(emus), status) for path, (emus, status) in files.items()]
|
||||||
|
|
||||||
|
|
||||||
def _normalize(destination: str, fold: bool = True) -> str:
|
def _normalize(destination: str, fold: bool = True) -> str:
|
||||||
"""Comparable form of a destination, since layers write it differently.
|
"""Comparable form of a destination, since layers write it differently.
|
||||||
@@ -661,11 +675,13 @@ def format_decision(decision: Decision) -> str:
|
|||||||
f"{conflict.destination}: serve {decision.winner.local_path}, "
|
f"{conflict.destination}: serve {decision.winner.local_path}, "
|
||||||
"the pack answers to the emulator"
|
"the pack answers to the emulator"
|
||||||
)
|
)
|
||||||
expected = ", ".join(conflict.emulators) or "the profile"
|
rejected = ", ".join(
|
||||||
|
f"{path}, the file {', '.join(emulators) or 'the profile'} reads"
|
||||||
|
for path, emulators, _status in conflict.files_read()
|
||||||
|
)
|
||||||
return (
|
return (
|
||||||
f"{conflict.destination}: keep {decision.winner.local_path}, the frontend "
|
f"{conflict.destination}: keep {decision.winner.local_path}, the frontend "
|
||||||
f"verifies content and would reject "
|
f"verifies content and would reject {rejected}"
|
||||||
f"{conflict.profile_claims[0].local_path}, the file {expected} reads"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -996,8 +1012,10 @@ def main() -> int:
|
|||||||
"ships": c.platform_claim.local_path,
|
"ships": c.platform_claim.local_path,
|
||||||
"ships_evidence": c.platform_claim.status,
|
"ships_evidence": c.platform_claim.status,
|
||||||
"emulators": c.emulators,
|
"emulators": c.emulators,
|
||||||
"expected": c.profile_claims[0].local_path,
|
"expected": [
|
||||||
"expected_evidence": c.profile_claims[0].status,
|
{"file": path, "evidence": status, "emulators": emulators}
|
||||||
|
for path, emulators, status in c.files_read()
|
||||||
|
],
|
||||||
}
|
}
|
||||||
for c in conflicts
|
for c in conflicts
|
||||||
]
|
]
|
||||||
|
|||||||
+78
-33
@@ -29,6 +29,7 @@ import contextlib
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
from collections import Counter
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -54,7 +55,6 @@ from common import (
|
|||||||
compute_hashes,
|
compute_hashes,
|
||||||
expand_directory_entries,
|
expand_directory_entries,
|
||||||
platform_declarations,
|
platform_declarations,
|
||||||
filter_systems_by_target,
|
|
||||||
group_identical_platforms,
|
group_identical_platforms,
|
||||||
list_emulator_profiles,
|
list_emulator_profiles,
|
||||||
list_system_ids,
|
list_system_ids,
|
||||||
@@ -63,7 +63,10 @@ from common import (
|
|||||||
load_emulator_profiles,
|
load_emulator_profiles,
|
||||||
load_platform_config,
|
load_platform_config,
|
||||||
resolve_system_arg,
|
resolve_system_arg,
|
||||||
|
print_platform_listing,
|
||||||
select_systems,
|
select_systems,
|
||||||
|
platform_scope_systems,
|
||||||
|
system_scope_error,
|
||||||
md5sum,
|
md5sum,
|
||||||
require_yaml,
|
require_yaml,
|
||||||
resolve_local_file,
|
resolve_local_file,
|
||||||
@@ -1217,16 +1220,7 @@ def verify_platform(
|
|||||||
validation_index = _build_validation_index(platform_profiles)
|
validation_index = _build_validation_index(platform_profiles)
|
||||||
validation_owners = destination_owners(platform_profiles)
|
validation_owners = destination_owners(platform_profiles)
|
||||||
|
|
||||||
if not target_cores:
|
verify_systems = platform_scope_systems(config, profiles, target_cores, system_filter)
|
||||||
plat_cores = None
|
|
||||||
verify_systems = filter_systems_by_target(
|
|
||||||
config.get("systems", {}),
|
|
||||||
profiles,
|
|
||||||
target_cores,
|
|
||||||
platform_cores=plat_cores,
|
|
||||||
)
|
|
||||||
if system_filter:
|
|
||||||
verify_systems = select_systems(verify_systems, system_filter)
|
|
||||||
|
|
||||||
details = []
|
details = []
|
||||||
file_status: dict[str, str] = {}
|
file_status: dict[str, str] = {}
|
||||||
@@ -1315,8 +1309,12 @@ def verify_platform(
|
|||||||
"release_asset": local_path,
|
"release_asset": local_path,
|
||||||
"reason": "release asset, fetched when the pack is built",
|
"reason": "release asset, fetched when the pack is built",
|
||||||
}
|
}
|
||||||
|
# Written at its path like any held file: keyed to None, it
|
||||||
|
# stayed out of pack_destinations and step 5 read the pack's
|
||||||
|
# copy as a file only the pack holds.
|
||||||
packed_order.append((
|
packed_order.append((
|
||||||
sys_id, f"{base_dest}/{destination}" if base_dest else destination, None,
|
sys_id, f"{base_dest}/{destination}" if base_dest else destination,
|
||||||
|
local_path,
|
||||||
))
|
))
|
||||||
shipped_keys.add(_dest_key(file_entry, sys_id))
|
shipped_keys.add(_dest_key(file_entry, sys_id))
|
||||||
local_path = None
|
local_path = None
|
||||||
@@ -1465,7 +1463,7 @@ def verify_platform(
|
|||||||
region_drops, report_for_builder,
|
region_drops, report_for_builder,
|
||||||
ExtraResolver(
|
ExtraResolver(
|
||||||
db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index,
|
db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, validation_index,
|
||||||
validation_owners, profiles=profiles,
|
validation_owners, profiles=profiles, pending_assets=True,
|
||||||
),
|
),
|
||||||
placed_kind,
|
placed_kind,
|
||||||
system_filter,
|
system_filter,
|
||||||
@@ -1746,13 +1744,20 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) ->
|
|||||||
print(f" {line}")
|
print(f" {line}")
|
||||||
|
|
||||||
|
|
||||||
def _print_undeclared_entry(u: dict, prefix: str, verbose: bool) -> None:
|
def _print_undeclared_entry(
|
||||||
"""Print a single undeclared file entry with its validation checks."""
|
u: dict, prefix: str, verbose: bool, repeated: frozenset[str] = frozenset()
|
||||||
|
) -> None:
|
||||||
|
"""Print a single undeclared file entry with its validation checks.
|
||||||
|
|
||||||
|
A name in ``repeated`` is read at more than one path, and the line names
|
||||||
|
the path: two identical lines told no one there were two slots to fill.
|
||||||
|
"""
|
||||||
|
name = u["path"] if u["name"] in repeated and u.get("path") else u["name"]
|
||||||
arc_count = u.get("archive_file_count")
|
arc_count = u.get("archive_file_count")
|
||||||
if arc_count:
|
if arc_count:
|
||||||
name_label = f"{u['name']} ({arc_count} file{'s' if arc_count != 1 else ''})"
|
name_label = f"{name} ({arc_count} file{'s' if arc_count != 1 else ''})"
|
||||||
else:
|
else:
|
||||||
name_label = u["name"]
|
name_label = name
|
||||||
print(f" {prefix}: {u['emulator']} needs {name_label}")
|
print(f" {prefix}: {u['emulator']} needs {name_label}")
|
||||||
checks = u.get("checks", [])
|
checks = u.get("checks", [])
|
||||||
if checks:
|
if checks:
|
||||||
@@ -1800,6 +1805,25 @@ def _core_required_missing(bios_files: list[dict]) -> list[dict]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _repeated_names(entries: list[dict]) -> frozenset[str]:
|
||||||
|
"""Names more than one of ``entries`` carries."""
|
||||||
|
counts = Counter(u["name"] for u in entries)
|
||||||
|
return frozenset(name for name, count in counts.items() if count > 1)
|
||||||
|
|
||||||
|
|
||||||
|
def _omission_reasons(entries: list[dict]) -> str:
|
||||||
|
"""Each reason a held file was left out, with its count when several.
|
||||||
|
|
||||||
|
Printing the first entry's reason for the whole count named one cause
|
||||||
|
for 412 RomM files when 20 carried it.
|
||||||
|
"""
|
||||||
|
counts = Counter(u.get("omitted", "not placed") for u in entries)
|
||||||
|
if len(counts) == 1:
|
||||||
|
return next(iter(counts))
|
||||||
|
ranked = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
|
||||||
|
return ", ".join(f"{count} {reason}" for reason, count in ranked)
|
||||||
|
|
||||||
|
|
||||||
def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
||||||
"""Print cross-reference section for undeclared files used by cores."""
|
"""Print cross-reference section for undeclared files used by cores."""
|
||||||
undeclared = result.get("undeclared_files", [])
|
undeclared = result.get("undeclared_files", [])
|
||||||
@@ -1839,19 +1863,22 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
|||||||
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, "
|
f" Core files: {core_in_pack} in pack, {core_missing_req} required missing, "
|
||||||
f"{core_missing_opt} optional missing{listed}"
|
f"{core_missing_opt} optional missing{listed}"
|
||||||
)
|
)
|
||||||
|
# The missing files follow the line that counts them: printed after the
|
||||||
|
# held-but-not-packed line, they read as files installed with their emulator.
|
||||||
|
repeated = _repeated_names(req_not_in_repo + req_hle_not_in_repo + opt_not_in_repo)
|
||||||
|
for u in req_not_in_repo:
|
||||||
|
_print_undeclared_entry(u, "MISSING (required)", verbose, repeated)
|
||||||
|
for u in req_hle_not_in_repo:
|
||||||
|
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose, repeated)
|
||||||
|
if verbose:
|
||||||
|
for u in opt_not_in_repo:
|
||||||
|
_print_undeclared_entry(u, "MISSING (optional)", verbose, repeated)
|
||||||
if bundled:
|
if bundled:
|
||||||
names = ", ".join(sorted({u["name"] for u in bundled}))
|
names = ", ".join(sorted({u["name"] for u in bundled}))
|
||||||
print(f" Bundled with their emulator, not collected: {names}")
|
print(f" Bundled with their emulator, not collected: {names}")
|
||||||
if unplaced:
|
if unplaced:
|
||||||
print(f" Core files held but not packed: {len(unplaced)} ({unplaced[0]['omitted']})")
|
print(f" Core files held but not packed: {len(unplaced)} "
|
||||||
|
f"({_omission_reasons(unplaced)})")
|
||||||
for u in req_not_in_repo:
|
|
||||||
_print_undeclared_entry(u, "MISSING (required)", verbose)
|
|
||||||
for u in req_hle_not_in_repo:
|
|
||||||
_print_undeclared_entry(u, "MISSING (required, HLE fallback)", verbose)
|
|
||||||
if verbose:
|
|
||||||
for u in opt_not_in_repo:
|
|
||||||
_print_undeclared_entry(u, "MISSING (optional)", verbose)
|
|
||||||
|
|
||||||
if game_data:
|
if game_data:
|
||||||
gd_missing = [u for u in game_data if not u["in_repo"]]
|
gd_missing = [u for u in game_data if not u["in_repo"]]
|
||||||
@@ -1864,12 +1891,13 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
|
|||||||
listed = "" if verbose or not gd_missing else " (--verbose lists them)"
|
listed = "" if verbose or not gd_missing else " (--verbose lists them)"
|
||||||
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}")
|
print(f" Game data: {gd_packed} in pack, {len(gd_missing)} missing{listed}")
|
||||||
if verbose:
|
if verbose:
|
||||||
|
repeated = _repeated_names(gd_missing)
|
||||||
for u in gd_missing:
|
for u in gd_missing:
|
||||||
_print_undeclared_entry(u, "MISSING (game data)", verbose)
|
_print_undeclared_entry(u, "MISSING (game data)", verbose, repeated)
|
||||||
if gd_unplaced:
|
if gd_unplaced:
|
||||||
print(
|
print(
|
||||||
f" Game data held but not packed: {len(gd_unplaced)} "
|
f" Game data held but not packed: {len(gd_unplaced)} "
|
||||||
f"({gd_unplaced[0].get('omitted', 'not placed')})"
|
f"({_omission_reasons(gd_unplaced)})"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -2597,6 +2625,7 @@ def _refuse_listing_narrowings(
|
|||||||
"""
|
"""
|
||||||
listing = next(
|
listing = next(
|
||||||
(flag for flag, on in (
|
(flag for flag, on in (
|
||||||
|
("--list-platforms", args.list_platforms),
|
||||||
("--list-emulators", args.list_emulators),
|
("--list-emulators", args.list_emulators),
|
||||||
("--list-systems", args.list_systems),
|
("--list-systems", args.list_systems),
|
||||||
("--list-targets", args.list_targets),
|
("--list-targets", args.list_targets),
|
||||||
@@ -2605,7 +2634,7 @@ def _refuse_listing_narrowings(
|
|||||||
)
|
)
|
||||||
if not listing:
|
if not listing:
|
||||||
return
|
return
|
||||||
reads_platform = listing != "--list-emulators"
|
reads_platform = listing not in ("--list-emulators", "--list-platforms")
|
||||||
for flag, on in (
|
for flag, on in (
|
||||||
("--platform", args.platform and not reads_platform),
|
("--platform", args.platform and not reads_platform),
|
||||||
("--all", args.all),
|
("--all", args.all),
|
||||||
@@ -2655,7 +2684,9 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
|
|||||||
|
|
||||||
|
|
||||||
def _run_listing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
def _run_listing(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
||||||
if args.list_emulators:
|
if args.list_platforms:
|
||||||
|
print_platform_listing(args.platforms_dir)
|
||||||
|
elif args.list_emulators:
|
||||||
list_emulator_profiles(args.emulators_dir)
|
list_emulator_profiles(args.emulators_dir)
|
||||||
elif args.list_systems and args.platform:
|
elif args.list_systems and args.platform:
|
||||||
list_platform_system_ids(args.platform, args.platforms_dir)
|
list_platform_system_ids(args.platform, args.platforms_dir)
|
||||||
@@ -2694,6 +2725,9 @@ def _main(held: contextlib.ExitStack) -> None:
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--list-systems", action="store_true", help="List available systems"
|
"--list-systems", action="store_true", help="List available systems"
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--list-platforms", action="store_true", help="List available platforms"
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--include-archived", action="store_true",
|
"--include-archived", action="store_true",
|
||||||
help="With --all, also verify the archived platforms (RetroPie)",
|
help="With --all, also verify the archived platforms (RetroPie)",
|
||||||
@@ -2767,7 +2801,7 @@ def _main(held: contextlib.ExitStack) -> None:
|
|||||||
parser.error(str(exc))
|
parser.error(str(exc))
|
||||||
|
|
||||||
_refuse_listing_narrowings(args, parser)
|
_refuse_listing_narrowings(args, parser)
|
||||||
if args.list_emulators or args.list_systems or args.list_targets:
|
if args.list_platforms or args.list_emulators or args.list_systems or args.list_targets:
|
||||||
_run_listing(args, parser)
|
_run_listing(args, parser)
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -2853,6 +2887,18 @@ def _main(held: contextlib.ExitStack) -> None:
|
|||||||
except (FileNotFoundError, ValueError) as e:
|
except (FileNotFoundError, ValueError) as e:
|
||||||
print(f"ERROR: {e}", file=sys.stderr)
|
print(f"ERROR: {e}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
# A system the target leaves no core for was reported 0/0 with exit 0,
|
||||||
|
# where generate_pack refuses the same request.
|
||||||
|
if platform_systems:
|
||||||
|
config = load_platform_config(args.platform, args.platforms_dir)
|
||||||
|
if not platform_scope_systems(
|
||||||
|
config, emu_profiles, target_cores_cache.get(args.platform), platform_systems
|
||||||
|
):
|
||||||
|
reason = system_scope_error(
|
||||||
|
config, args.platform, platform_systems, args.target
|
||||||
|
)
|
||||||
|
print(f"ERROR: {reason}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
# Group identical platforms (same function as generate_pack)
|
# Group identical platforms (same function as generate_pack)
|
||||||
groups = group_identical_platforms(
|
groups = group_identical_platforms(
|
||||||
@@ -2961,8 +3007,7 @@ def _overall_summary(group_results: list[tuple[dict, list[str]]]) -> str:
|
|||||||
platforms = sum(len(group) for _, group in group_results)
|
platforms = sum(len(group) for _, group in group_results)
|
||||||
|
|
||||||
def where(counts: dict[str, int]) -> str:
|
def where(counts: dict[str, int]) -> str:
|
||||||
named = [f"{name} {count}" if count > 1 else name
|
named = [f"{name} {count}" for name, count in counts.items() if count]
|
||||||
for name, count in counts.items() if count]
|
|
||||||
return f" ({', '.join(named)})" if named else ""
|
return f" ({', '.join(named)})" if named else ""
|
||||||
|
|
||||||
parts = []
|
parts = []
|
||||||
|
|||||||
@@ -4830,7 +4830,7 @@ class VerifyAllSummaryNamesWhatIsWrong(unittest.TestCase):
|
|||||||
summary = verify._overall_summary(
|
summary = verify._overall_summary(
|
||||||
[(clean, ["RetroArch"]), (warned, ["RetroDECK"]), (core_miss, ["ROCKNIX"])]
|
[(clean, ["RetroArch"]), (warned, ["RetroDECK"]), (core_miss, ["ROCKNIX"])]
|
||||||
)
|
)
|
||||||
self.assertIn("1 warning (RetroDECK)", summary)
|
self.assertIn("1 warning (RetroDECK 1)", summary)
|
||||||
self.assertIn("ROCKNIX: n64.psh", summary)
|
self.assertIn("ROCKNIX: n64.psh", summary)
|
||||||
self.assertIn("0 critical,", summary)
|
self.assertIn("0 critical,", summary)
|
||||||
|
|
||||||
|
|||||||
@@ -87,6 +87,59 @@ class CoreFilesAsTheBuilderPlacesThem(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ARefusedPinLeavesItsPathToTheCore(unittest.TestCase):
|
||||||
|
"""EmuELEC pins naomi.zip by an md5 the collection does not hold: the
|
||||||
|
builder refuses the held copy and ships the core's file at that path.
|
||||||
|
The check marked the path taken by the refused platform file, so the
|
||||||
|
core file the pack carries there was never compared."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
tmp = tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp")
|
||||||
|
self.addCleanup(tmp.cleanup)
|
||||||
|
previous = os.getcwd()
|
||||||
|
os.chdir(tmp.name)
|
||||||
|
self.addCleanup(os.chdir, previous)
|
||||||
|
files: dict = {}
|
||||||
|
by_name: dict = {}
|
||||||
|
_held(files, by_name, "bios/dc/naomi.zip", b"the core's naomi")
|
||||||
|
self.db = {"files": files, "indexes": {
|
||||||
|
"by_name": by_name, "by_md5": {e["md5"]: s for s, e in files.items()},
|
||||||
|
"by_path_suffix": {}, "by_crc32": {}, "by_sha256": {}}}
|
||||||
|
Path("platforms").mkdir()
|
||||||
|
config = {"platform": "P", "verification_mode": "md5", "cores": ["flycast"],
|
||||||
|
"systems": {"dc": {"files": [{
|
||||||
|
"name": "naomi.zip", "destination": "dc/naomi.zip",
|
||||||
|
"md5": hashlib.md5(b"a bundle nobody holds").hexdigest(),
|
||||||
|
}]}}}
|
||||||
|
Path("platforms/p.yml").write_text(yaml.safe_dump(config), encoding="utf-8")
|
||||||
|
Path("emulators").mkdir()
|
||||||
|
self.profiles = {"flycast": {
|
||||||
|
"emulator": "flycast", "type": "libretro", "systems": ["dc"], "files": [
|
||||||
|
{"name": "naomi.zip", "path": "dc/naomi.zip"}]}}
|
||||||
|
|
||||||
|
def _check(self, member: bytes) -> tuple:
|
||||||
|
with zipfile.ZipFile("pack.zip", "w") as zf:
|
||||||
|
zf.writestr("dc/naomi.zip", member)
|
||||||
|
return packverify.verify_pack_against_platform(
|
||||||
|
"pack.zip", "p", "platforms", self.db, emulators_dir="emulators",
|
||||||
|
emu_profiles=self.profiles,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_the_core_file_at_the_refused_path_is_compared(self):
|
||||||
|
_ok, _c, _p, errors, *_mid, core_checked, _cp, _ex, _ab = self._check(b"other")
|
||||||
|
self.assertEqual(core_checked, 1)
|
||||||
|
self.assertTrue(
|
||||||
|
any(e.startswith("core file differs: dc/naomi.zip") for e in errors), errors
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_the_right_core_file_passes(self):
|
||||||
|
_ok, _c, _p, errors, *_mid, core_checked, core_present, _ex, _ab = (
|
||||||
|
self._check(b"the core's naomi")
|
||||||
|
)
|
||||||
|
self.assertEqual((core_checked, core_present), (1, 1))
|
||||||
|
self.assertFalse([e for e in errors if e.startswith("core")], errors)
|
||||||
|
|
||||||
|
|
||||||
class NarrowedPacksAgainstTheirOwnList(unittest.TestCase):
|
class NarrowedPacksAgainstTheirOwnList(unittest.TestCase):
|
||||||
"""--verify-packs after --all-variants said "hashes checked during the
|
"""--verify-packs after --all-variants said "hashes checked during the
|
||||||
build" for the five narrowed packs and compared none of them: a narrowed
|
build" for the five narrowed packs and compared none of them: a narrowed
|
||||||
|
|||||||
@@ -420,6 +420,24 @@ class TestArbitration(unittest.TestCase):
|
|||||||
served = slots.format_decision(slots.arbitrate(self._conflict(), "existence"))
|
served = slots.format_decision(slots.arbitrate(self._conflict(), "existence"))
|
||||||
self.assertIn("both are satisfied", served)
|
self.assertIn("both are satisfied", served)
|
||||||
|
|
||||||
|
def test_each_emulator_is_given_the_file_it_reads(self):
|
||||||
|
"""Recalbox's neocdz.zip line said geolith reads the archive fbneo
|
||||||
|
reads, while geolith pins another one."""
|
||||||
|
conflict = self._conflict()
|
||||||
|
conflict.profile_claims = [
|
||||||
|
slots.Claim("profile", "neocdz.zip", "neocdz.zip", emulator="fbneo",
|
||||||
|
local_path="a/neocdz.zip.ba20a7be", status="members_held"),
|
||||||
|
slots.Claim("profile", "neocdz.zip", "neocdz.zip", emulator="geolith",
|
||||||
|
local_path="b/neocdz.zip.712816a3", status="md5_exact"),
|
||||||
|
]
|
||||||
|
kept = slots.format_decision(slots.arbitrate(conflict, "md5"))
|
||||||
|
self.assertIn("a/neocdz.zip.ba20a7be, the file fbneo reads", kept)
|
||||||
|
self.assertIn("b/neocdz.zip.712816a3, the file geolith reads", kept)
|
||||||
|
self.assertEqual(conflict.files_read(), [
|
||||||
|
("a/neocdz.zip.ba20a7be", ["fbneo"], "members_held"),
|
||||||
|
("b/neocdz.zip.712816a3", ["geolith"], "md5_exact"),
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
class TheStrongestProofTakesAnUndeclaredPath(unittest.TestCase):
|
class TheStrongestProofTakesAnUndeclaredPath(unittest.TestCase):
|
||||||
"""No platform declares RetroDECK's hiscore.dat. fbalpha2012 asks for it
|
"""No platform declares RetroDECK's hiscore.dat. fbalpha2012 asks for it
|
||||||
|
|||||||
+169
-2
@@ -4,11 +4,13 @@
|
|||||||
under the target's name (Recalbox, Lakka, BizHawk, MiSTer, ROCKNIX). The
|
under the target's name (Recalbox, Lakka, BizHawk, MiSTer, ROCKNIX). The
|
||||||
conformance check read the same target without the --all tolerance, raised
|
conformance check read the same target without the --all tolerance, raised
|
||||||
FileNotFoundError on the first of those packs and stopped the run in a
|
FileNotFoundError on the first of those packs and stopped the run in a
|
||||||
traceback after every pack had been written.
|
traceback after every pack had been written. Such a platform is now left
|
||||||
|
out of a targeted --all run, and the check refuses it as the build does.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
import sys
|
import sys
|
||||||
import contextlib
|
import contextlib
|
||||||
import io
|
import io
|
||||||
@@ -31,7 +33,10 @@ class CheckReadsTheTargetLikeTheBuild(unittest.TestCase):
|
|||||||
contextlib.redirect_stdout(io.StringIO()):
|
contextlib.redirect_stdout(io.StringIO()):
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
build_target_cores_cache(["recalbox"], "x86_64", platforms, is_all=True)
|
build_target_cores_cache(["recalbox"], "x86_64", platforms, is_all=True)
|
||||||
self.assertIsNone(_target_cores_for("recalbox", "x86_64", platforms))
|
# The build never packs such a platform under the target, so the
|
||||||
|
# check refuses it too instead of reading the full pack.
|
||||||
|
with self.assertRaises(FileNotFoundError):
|
||||||
|
_target_cores_for("recalbox", "x86_64", platforms)
|
||||||
|
|
||||||
def test_a_platform_with_a_target_file_is_narrowed(self):
|
def test_a_platform_with_a_target_file_is_narrowed(self):
|
||||||
cores = _target_cores_for("retroarch", "switch", str(REPO_ROOT / "platforms"))
|
cores = _target_cores_for("retroarch", "switch", str(REPO_ROOT / "platforms"))
|
||||||
@@ -54,5 +59,167 @@ class ANameClaimsOnlyANarrowingThatHappened(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AMistypedTargetIsRefusedFirst(unittest.TestCase):
|
||||||
|
"""`--platform retroarch --target swich` took the dist lock and waited on
|
||||||
|
the data directory refresh, 22 s here and hours behind a pipeline, before
|
||||||
|
saying the target does not exist."""
|
||||||
|
|
||||||
|
def test_no_lock_and_no_refresh_before_the_refusal(self):
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
import generate_pack
|
||||||
|
|
||||||
|
def touched(*_a, **_kw):
|
||||||
|
raise AssertionError("lock or refresh taken before the target check")
|
||||||
|
|
||||||
|
argv = ["generate_pack.py", "--platform", "retroarch", "--target", "swich",
|
||||||
|
"--output-dir", str(REPO_ROOT / "tmp" / "never-written")]
|
||||||
|
err = io.StringIO()
|
||||||
|
with mock.patch.object(sys, "argv", argv), \
|
||||||
|
mock.patch.object(generate_pack, "_pack_output_lock", touched), \
|
||||||
|
mock.patch.object(generate_pack, "hold_data_dirs", touched), \
|
||||||
|
contextlib.redirect_stdout(io.StringIO()), \
|
||||||
|
contextlib.redirect_stderr(err), \
|
||||||
|
self.assertRaises(SystemExit) as ctx:
|
||||||
|
generate_pack.main()
|
||||||
|
self.assertEqual(ctx.exception.code, 1)
|
||||||
|
self.assertIn("Unknown target 'swich'", err.getvalue())
|
||||||
|
|
||||||
|
|
||||||
|
class ASystemTheTargetDropsIsNamedAsSuch(unittest.TestCase):
|
||||||
|
"""`--platform retroarch --system sony-playstation-2 --target switch`:
|
||||||
|
verify printed 0/0 with exit 0 under --strict, and generate_pack said
|
||||||
|
RetroArch has no such system and suggested sony-playstation."""
|
||||||
|
|
||||||
|
PLATFORMS = str(REPO_ROOT / "platforms")
|
||||||
|
|
||||||
|
def test_both_tools_read_one_scope(self):
|
||||||
|
from common import (
|
||||||
|
load_emulator_profiles, load_platform_config, platform_scope_systems,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = load_platform_config("retroarch", self.PLATFORMS)
|
||||||
|
profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
|
||||||
|
switch = _target_cores_for("retroarch", "switch", self.PLATFORMS)
|
||||||
|
wanted = ["sony-playstation-2"]
|
||||||
|
self.assertEqual(platform_scope_systems(config, profiles, switch, wanted), {})
|
||||||
|
self.assertEqual(list(platform_scope_systems(config, profiles, None, wanted)), wanted)
|
||||||
|
|
||||||
|
def test_the_message_names_the_target(self):
|
||||||
|
from common import system_scope_error
|
||||||
|
|
||||||
|
config = {"systems": {"sony-playstation-2": {}, "sony-playstation": {}}}
|
||||||
|
self.assertEqual(
|
||||||
|
system_scope_error(config, "retroarch", ["sony-playstation-2"], "switch"),
|
||||||
|
"no core retroarch runs on target switch emulates sony-playstation-2",
|
||||||
|
)
|
||||||
|
undeclared = system_scope_error(config, "retroarch", ["nec-pc-98"], "switch")
|
||||||
|
self.assertTrue(undeclared.startswith("retroarch has no system matching nec-pc-98."))
|
||||||
|
|
||||||
|
def test_verify_refuses_it(self):
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
proc = subprocess.run(
|
||||||
|
[sys.executable, "scripts/verify.py", "--platform", "retroarch",
|
||||||
|
"--system", "sony-playstation-2", "--target", "switch", "--strict"],
|
||||||
|
capture_output=True, text=True, cwd=str(REPO_ROOT), timeout=600, check=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(proc.returncode, 1, proc.stdout[-300:])
|
||||||
|
self.assertIn("no core retroarch runs on target switch emulates sony-playstation-2",
|
||||||
|
proc.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
class ASplitBuildsOnlyTheTargetsSystems(unittest.TestCase):
|
||||||
|
"""`--split --target switch` asked for a part per declared system, and
|
||||||
|
each one the target drops ended on 'retroarch has no system matching'."""
|
||||||
|
|
||||||
|
def test_no_part_is_asked_for_a_dropped_system(self):
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
import generate_pack
|
||||||
|
from common import load_emulator_profiles
|
||||||
|
|
||||||
|
platforms = str(REPO_ROOT / "platforms")
|
||||||
|
profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
|
||||||
|
asked: list[list[str]] = []
|
||||||
|
|
||||||
|
def record(*_a, **kw):
|
||||||
|
if kw.get("system_filter"):
|
||||||
|
asked.append(kw["system_filter"])
|
||||||
|
return None
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \
|
||||||
|
mock.patch.object(generate_pack, "generate_pack", record), \
|
||||||
|
mock.patch.object(generate_pack, "_collect_emulator_extras",
|
||||||
|
return_value=[]):
|
||||||
|
generate_pack.generate_split_packs(
|
||||||
|
"retroarch", platforms, {"files": {}}, "bios", out,
|
||||||
|
emu_profiles=profiles,
|
||||||
|
target_cores=_target_cores_for("retroarch", "switch", platforms),
|
||||||
|
target_name="nintendo-switch",
|
||||||
|
)
|
||||||
|
systems = {sid for ids in asked for sid in ids}
|
||||||
|
self.assertTrue(systems)
|
||||||
|
self.assertNotIn("sony-playstation-2", systems)
|
||||||
|
self.assertIn("sony-playstation", systems)
|
||||||
|
|
||||||
|
|
||||||
|
class APlatformWithoutTheSystemIsOutOfScope(unittest.TestCase):
|
||||||
|
"""`--all --system sony-playstation-2` built the five PS2 packs and
|
||||||
|
exited 1 on 'no pack for misterfpga', which has no PS2."""
|
||||||
|
|
||||||
|
def _scope(self, **args):
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
import generate_pack
|
||||||
|
from common import load_emulator_profiles
|
||||||
|
|
||||||
|
values = {"platforms_dir": str(REPO_ROOT / "platforms"), "target": None,
|
||||||
|
"all": True}
|
||||||
|
values.update(args)
|
||||||
|
out = io.StringIO()
|
||||||
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
|
||||||
|
kept = generate_pack._platforms_with_systems(
|
||||||
|
argparse.Namespace(**values), ["retroarch", "misterfpga"],
|
||||||
|
["sony-playstation-2"],
|
||||||
|
load_emulator_profiles(str(REPO_ROOT / "emulators")), {},
|
||||||
|
)
|
||||||
|
return kept, out.getvalue()
|
||||||
|
|
||||||
|
def test_all_skips_it(self):
|
||||||
|
kept, out = self._scope()
|
||||||
|
self.assertEqual(kept, ["retroarch"])
|
||||||
|
self.assertIn("INFO: Skipping misterfpga: no system sony-playstation-2", out)
|
||||||
|
|
||||||
|
def test_a_named_platform_is_refused(self):
|
||||||
|
with self.assertRaises(SystemExit) as ctx:
|
||||||
|
self._scope(all=False)
|
||||||
|
self.assertEqual(ctx.exception.code, 1)
|
||||||
|
|
||||||
|
|
||||||
|
class TheSplitDirectoryReadsThePackStem(unittest.TestCase):
|
||||||
|
"""The split prefix was assembled by hand: a version 'release/1.22' made
|
||||||
|
'RetroArch_release/1.22_Split', a nested directory the check never
|
||||||
|
found, beside a full pack named 'RetroArch_release-1.22_BIOS_Pack.zip'."""
|
||||||
|
|
||||||
|
def test_a_reserved_character_is_made_safe_as_in_the_full_pack(self):
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
import generate_pack
|
||||||
|
|
||||||
|
config = {"platform": "Foo: Bar", "version": "release/1.22", "systems": {}}
|
||||||
|
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \
|
||||||
|
mock.patch.object(generate_pack, "load_platform_config",
|
||||||
|
return_value=config), \
|
||||||
|
mock.patch.object(generate_pack, "_collect_emulator_extras",
|
||||||
|
return_value=[]):
|
||||||
|
generate_pack.generate_split_packs(
|
||||||
|
"foo", "platforms", {"files": {}}, "bios", out, emu_profiles={},
|
||||||
|
)
|
||||||
|
stem = generate_pack._platform_pack_stem(["foo"], "foo", "platforms")
|
||||||
|
self.assertEqual(os.listdir(out), [f"{stem}_Split"])
|
||||||
|
self.assertEqual(stem, "Foo-_Bar_release-1.22")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
+116
-1
@@ -132,7 +132,7 @@ class TheVerifySummaryCountsDiscrepancies(unittest.TestCase):
|
|||||||
({"severity_counts": {}, "details": details}, ["RetroArch"]),
|
({"severity_counts": {}, "details": details}, ["RetroArch"]),
|
||||||
({"severity_counts": {}, "details": []}, ["Batocera"]),
|
({"severity_counts": {}, "details": []}, ["Batocera"]),
|
||||||
])
|
])
|
||||||
self.assertIn("1 discrepancy (RetroArch)", summary)
|
self.assertIn("1 discrepancy (RetroArch 1)", summary)
|
||||||
|
|
||||||
def test_a_shared_destination_names_the_path_and_both_sides(self):
|
def test_a_shared_destination_names_the_path_and_both_sides(self):
|
||||||
result = {"name": "disk.rom", "status": verify.Status.UNTESTED}
|
result = {"name": "disk.rom", "status": verify.Status.UNTESTED}
|
||||||
@@ -145,6 +145,121 @@ class TheVerifySummaryCountsDiscrepancies(unittest.TestCase):
|
|||||||
self.assertIn(part, message)
|
self.assertIn(part, message)
|
||||||
|
|
||||||
|
|
||||||
|
class EveryPlatformInTheSummaryCarriesItsCount(unittest.TestCase):
|
||||||
|
"""'17 discrepancies (Lakka 5, RetroArch 5, RetroBat 2, RetroDECK 4,
|
||||||
|
ROCKNIX)': the one platform with a single entry had no number, and the
|
||||||
|
named counts did not add up to the total."""
|
||||||
|
|
||||||
|
def test_a_count_of_one_is_written(self):
|
||||||
|
one = {"severity_counts": {verify.Severity.WARNING: 1}, "details": []}
|
||||||
|
two = {"severity_counts": {verify.Severity.WARNING: 2}, "details": []}
|
||||||
|
summary = verify._overall_summary([(two, ["Lakka"]), (one, ["ROCKNIX"])])
|
||||||
|
self.assertIn("3 warning (Lakka 2, ROCKNIX 1)", summary)
|
||||||
|
|
||||||
|
|
||||||
|
class HeldButNotPackedNamesEachReason(unittest.TestCase):
|
||||||
|
"""'Core files held but not packed: 16 (a packed file occupies its
|
||||||
|
directory)' on Lakka, where 14 of the 16 are installed with their
|
||||||
|
emulator: the first entry's reason stood for the whole count."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _entry(name: str, reason: str, category: str = "bios") -> dict:
|
||||||
|
return {"name": name, "emulator": "x", "required": True, "in_repo": True,
|
||||||
|
"in_pack": False, "omitted": reason, "category": category}
|
||||||
|
|
||||||
|
def _printed(self, entries: list[dict]) -> str:
|
||||||
|
out = io.StringIO()
|
||||||
|
with contextlib.redirect_stdout(out):
|
||||||
|
verify._print_undeclared_section({"undeclared_files": entries}, False)
|
||||||
|
return out.getvalue()
|
||||||
|
|
||||||
|
def test_each_reason_carries_its_count(self):
|
||||||
|
entries = [self._entry("a.rom", "a packed file occupies its directory")]
|
||||||
|
entries += [self._entry(f"{i}.rom", "installed with its emulator") for i in range(3)]
|
||||||
|
entries += [self._entry(f"g{i}", "installed with its emulator", "game_data")
|
||||||
|
for i in range(2)]
|
||||||
|
entries.append(self._entry("g9", "no platform slug for its system", "game_data"))
|
||||||
|
text = self._printed(entries)
|
||||||
|
self.assertIn(
|
||||||
|
"Core files held but not packed: 4 (3 installed with its emulator, "
|
||||||
|
"1 a packed file occupies its directory)", text)
|
||||||
|
self.assertIn(
|
||||||
|
"Game data held but not packed: 3 (2 installed with its emulator, "
|
||||||
|
"1 no platform slug for its system)", text)
|
||||||
|
|
||||||
|
def test_a_missing_file_follows_the_line_that_counts_it(self):
|
||||||
|
"""Indented under 'held but not packed: 102 (installed with its
|
||||||
|
emulator)', Batocera's one missing sfiii2a.zip read as one of them."""
|
||||||
|
missing = {"name": "sfiii2a.zip", "emulator": "FinalBurn Neo", "required": True,
|
||||||
|
"in_repo": False, "category": "bios"}
|
||||||
|
lines = self._printed(
|
||||||
|
[missing, self._entry("a.rom", "installed with its emulator")]
|
||||||
|
).splitlines()
|
||||||
|
self.assertTrue(lines[0].startswith(" Core files: "), lines)
|
||||||
|
self.assertIn("MISSING (required): FinalBurn Neo needs sfiii2a.zip", lines[1])
|
||||||
|
self.assertTrue(lines[2].startswith(" Core files held but not packed: "), lines)
|
||||||
|
|
||||||
|
def test_one_name_read_at_two_paths_prints_both_paths(self):
|
||||||
|
"""MyMinUI printed 'FinalBurn Neo needs sfiii2a.zip' twice, word for
|
||||||
|
word, for Bios/FBN/ and Bios/NG/."""
|
||||||
|
base = {"name": "sfiii2a.zip", "emulator": "FinalBurn Neo", "required": True,
|
||||||
|
"in_repo": False, "category": "bios"}
|
||||||
|
lone = dict(base, name="other.zip", path="Bios/FBN/other.zip")
|
||||||
|
text = self._printed([dict(base, path="Bios/FBN/sfiii2a.zip"),
|
||||||
|
dict(base, path="Bios/NG/sfiii2a.zip"), lone])
|
||||||
|
self.assertIn("FinalBurn Neo needs Bios/FBN/sfiii2a.zip", text)
|
||||||
|
self.assertIn("FinalBurn Neo needs Bios/NG/sfiii2a.zip", text)
|
||||||
|
self.assertIn("FinalBurn Neo needs other.zip\n", text)
|
||||||
|
|
||||||
|
def test_a_single_reason_reads_as_before(self):
|
||||||
|
text = self._printed([self._entry("a.rom", "installed with its emulator")] * 2)
|
||||||
|
self.assertIn("Core files held but not packed: 2 (installed with its emulator)", text)
|
||||||
|
|
||||||
|
|
||||||
|
class APlatformWithoutTargetsSaysSo(unittest.TestCase):
|
||||||
|
"""`--platform misterfpga --target switch` printed 'No target config for
|
||||||
|
platform 'misterfpga': platforms/targets/misterfpga.yml', which reads as
|
||||||
|
a file missing from the repository."""
|
||||||
|
|
||||||
|
def test_the_message_names_the_reason_and_the_platforms_with_targets(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
(Path(tmp) / "targets").mkdir()
|
||||||
|
for name in ("retroarch", "batocera", "_overrides"):
|
||||||
|
(Path(tmp) / "targets" / f"{name}.yml").write_text("targets: {}\n")
|
||||||
|
with self.assertRaises(FileNotFoundError) as caught:
|
||||||
|
common.load_target_config("misterfpga", "switch", tmp)
|
||||||
|
self.assertEqual(
|
||||||
|
str(caught.exception),
|
||||||
|
"misterfpga has no hardware targets: drop --target "
|
||||||
|
"(platforms with targets: batocera, retroarch)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class OneNameListsThePlatforms(unittest.TestCase):
|
||||||
|
"""install.py --list-platforms, generate_pack.py --list, and no listing
|
||||||
|
at all in verify.py: three scripts, three answers to one question."""
|
||||||
|
|
||||||
|
def _run(self, script: str, flag: str) -> subprocess.CompletedProcess:
|
||||||
|
return subprocess.run(
|
||||||
|
[sys.executable, f"scripts/{script}", flag], capture_output=True, text=True,
|
||||||
|
cwd=str(REPO_ROOT), timeout=120, check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_both_tools_take_it_and_agree(self):
|
||||||
|
verify_out = self._run("verify.py", "--list-platforms")
|
||||||
|
pack_out = self._run("generate_pack.py", "--list-platforms")
|
||||||
|
self.assertEqual(verify_out.returncode, 0, verify_out.stderr)
|
||||||
|
self.assertEqual(pack_out.returncode, 0, pack_out.stderr)
|
||||||
|
self.assertEqual(verify_out.stdout, pack_out.stdout)
|
||||||
|
self.assertIn("retroarch", verify_out.stdout.split())
|
||||||
|
self.assertIn("--list-platforms", (REPO_ROOT / "install.py").read_text())
|
||||||
|
|
||||||
|
def test_the_installer_name_is_the_one_documented(self):
|
||||||
|
tools = (REPO_ROOT / "wiki" / "tools.md").read_text()
|
||||||
|
self.assertIn("generate_pack.py --list-platforms", tools)
|
||||||
|
self.assertIn("verify.py --list-platforms", tools)
|
||||||
|
|
||||||
|
|
||||||
class OneSpellingPerOption(unittest.TestCase):
|
class OneSpellingPerOption(unittest.TestCase):
|
||||||
"""verify.py -p batocera worked and slots.py -p batocera was refused."""
|
"""verify.py -p batocera worked and slots.py -p batocera was refused."""
|
||||||
|
|
||||||
|
|||||||
@@ -116,6 +116,24 @@ class VerifyPackChoice(unittest.TestCase):
|
|||||||
dict(zip((name for _p, name in checked), self.variants)), names
|
dict(zip((name for _p, name in checked), self.variants)), names
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_a_target_the_platform_lacks_is_refused(self):
|
||||||
|
"""`--platform bizhawk --target rpi4typo --verify-packs` verified the
|
||||||
|
full pack and said `all OK`, where the build refused the target."""
|
||||||
|
import contextlib
|
||||||
|
import io
|
||||||
|
|
||||||
|
bizhawk = self.gp._platform_pack_stem(["bizhawk"], "bizhawk", "platforms")
|
||||||
|
self._touch(f"{bizhawk}_BIOS_Pack.zip")
|
||||||
|
self._touch(f"{self.stem}_BIOS_Pack.zip")
|
||||||
|
for platform, target in (("bizhawk", "rpi4typo"), ("retroarch", "swich")):
|
||||||
|
with self.subTest(platform=platform), \
|
||||||
|
contextlib.redirect_stdout(io.StringIO()), \
|
||||||
|
contextlib.redirect_stderr(io.StringIO()), \
|
||||||
|
self.assertRaises(SystemExit) as ctx:
|
||||||
|
self._run(self._args(platform=platform, target=target))
|
||||||
|
self.assertEqual(ctx.exception.code, 1)
|
||||||
|
self.assertEqual(self.variants, [])
|
||||||
|
|
||||||
def test_the_run_ends_on_a_summary(self):
|
def test_the_run_ends_on_a_summary(self):
|
||||||
"""A failing pack was named only above its own error lines, and the
|
"""A failing pack was named only above its own error lines, and the
|
||||||
log ended on the last platform's OK line."""
|
log ended on the last platform's OK line."""
|
||||||
|
|||||||
@@ -89,6 +89,12 @@ class ReleaseAssetNotCached(unittest.TestCase):
|
|||||||
self.assertEqual(entry["status"], Status.OK)
|
self.assertEqual(entry["status"], Status.OK)
|
||||||
self.assertEqual(entry["release_asset"], REGISTERED[0])
|
self.assertEqual(entry["release_asset"], REGISTERED[0])
|
||||||
|
|
||||||
|
def test_its_path_is_a_pack_destination(self):
|
||||||
|
"""The pack holds the fetched file at its path: absent from
|
||||||
|
pack_destinations, step 5 reported it 'pack only'."""
|
||||||
|
report = verify_platform(self.config, self.db, "emulators", {})
|
||||||
|
self.assertIn("big.bin", report["pack_destinations"])
|
||||||
|
|
||||||
def test_offline_counts_it_missing(self):
|
def test_offline_counts_it_missing(self):
|
||||||
packresolve.set_offline(True)
|
packresolve.set_offline(True)
|
||||||
self.assertEqual(self._entry()["status"], Status.MISSING)
|
self.assertEqual(self._entry()["status"], Status.MISSING)
|
||||||
@@ -117,6 +123,59 @@ class ReleaseAssetNotCachedMd5(ReleaseAssetNotCached):
|
|||||||
mode = "md5"
|
mode = "md5"
|
||||||
|
|
||||||
|
|
||||||
|
class CoreExtraReleaseAssetNotCached(ReleaseAssetNotCached):
|
||||||
|
"""The same asset read by a core and declared by no platform entry: the
|
||||||
|
builder fetched and packed it online, verify's replay of the extras
|
||||||
|
resolved it offline and left its path out of the pack."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super().setUp()
|
||||||
|
self.config["systems"]["sys"]["files"] = []
|
||||||
|
self.config["cores"] = ["bigcore"]
|
||||||
|
Path("platforms/big.yml").write_text(yaml.dump(self.config))
|
||||||
|
self.profiles = {"bigcore": {
|
||||||
|
"emulator": "Big Core", "type": "libretro", "systems": ["sys"],
|
||||||
|
"files": [{"name": "big.bin", "required": True, "sha1": SHA1}],
|
||||||
|
}}
|
||||||
|
Path("emulators/bigcore.yml").write_text(yaml.dump(self.profiles["bigcore"]))
|
||||||
|
common._platform_config_cache.clear()
|
||||||
|
common._emulator_profiles_cache.clear()
|
||||||
|
|
||||||
|
def _report(self) -> dict:
|
||||||
|
return verify_platform(self.config, self.db, "emulators", self.profiles)
|
||||||
|
|
||||||
|
def test_counted_as_fetched_when_the_pack_is_built(self):
|
||||||
|
report = self._report()
|
||||||
|
self.assertIn("big.bin", report["pack_destinations"])
|
||||||
|
self.assertEqual(report["packed_extras"]["bios"], 1)
|
||||||
|
|
||||||
|
def test_its_path_is_a_pack_destination(self):
|
||||||
|
cache = Path("cache/big.bin")
|
||||||
|
cache.parent.mkdir()
|
||||||
|
cache.write_bytes(PAYLOAD)
|
||||||
|
with mock.patch.object(
|
||||||
|
packresolve, "fetch_large_file", return_value=str(cache)
|
||||||
|
):
|
||||||
|
zip_path = builder.generate_pack(
|
||||||
|
"big", "platforms", self.db, "bios", "out",
|
||||||
|
emulators_dir="emulators", emu_profiles=self.profiles, offline=False,
|
||||||
|
)
|
||||||
|
with zipfile.ZipFile(zip_path) as zf:
|
||||||
|
packed = {n for n in zf.namelist() if not n.endswith("/")}
|
||||||
|
self.assertIn("big.bin", packed)
|
||||||
|
self.assertIn("big.bin", self._report()["pack_destinations"])
|
||||||
|
|
||||||
|
def test_offline_counts_it_missing(self):
|
||||||
|
packresolve.set_offline(True)
|
||||||
|
self.assertNotIn("big.bin", self._report()["pack_destinations"])
|
||||||
|
|
||||||
|
def test_an_asset_with_other_content_is_not_counted(self):
|
||||||
|
self.profiles["bigcore"]["files"][0]["sha1"] = "0" * 40
|
||||||
|
self.assertNotIn("big.bin", self._report()["pack_destinations"])
|
||||||
|
|
||||||
|
test_the_online_build_packs_it = None
|
||||||
|
|
||||||
|
|
||||||
class OfflinePipelineVerifiesOffline(unittest.TestCase):
|
class OfflinePipelineVerifiesOffline(unittest.TestCase):
|
||||||
"""The offline pipeline builds packs without release assets: its verify
|
"""The offline pipeline builds packs without release assets: its verify
|
||||||
step has to count them the same way, or step 5 compares two answers."""
|
step has to count them the same way, or step 5 compares two answers."""
|
||||||
|
|||||||
@@ -117,7 +117,8 @@ RetroDECK: 2007/2008 OK, 1 untested [md5]
|
|||||||
|
|
||||||
The run prints more `DISCREPANCY` lines than this excerpt shows. The core
|
The run prints more `DISCREPANCY` lines than this excerpt shows. The core
|
||||||
and game data lines count what the emulator cores add beyond the platform's
|
and game data lines count what the emulator cores add beyond the platform's
|
||||||
own list. A file held but not packed is left out for the reason in parentheses.
|
own list. A file held but not packed is left out for the reason in parentheses, each
|
||||||
|
reason with its count when the files have several.
|
||||||
|
|
||||||
`--verbose` also lists the passing entries that an emulator validates, with
|
`--verbose` also lists the passing entries that an emulator validates, with
|
||||||
the check and the source lines behind it:
|
the check and the source lines behind it:
|
||||||
|
|||||||
+2
-1
@@ -134,6 +134,7 @@ python scripts/verify.py --platform recalbox --region us # regional priority l
|
|||||||
python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named
|
python scripts/verify.py --platform emuelec --no-game-data # the firmware-only pack, game data named
|
||||||
python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds
|
python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds
|
||||||
python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build
|
python scripts/verify.py --all --offline # a release asset the cache lacks counts missing, as in an offline build
|
||||||
|
python scripts/verify.py --list-platforms # list all platforms
|
||||||
python scripts/verify.py --list-emulators # list all emulators
|
python scripts/verify.py --list-emulators # list all emulators
|
||||||
python scripts/verify.py --list-systems # list all systems
|
python scripts/verify.py --list-systems # list all systems
|
||||||
python scripts/verify.py --platform retroarch --list-targets # list available targets
|
python scripts/verify.py --platform retroarch --list-targets # list available targets
|
||||||
@@ -208,7 +209,7 @@ python scripts/generate_pack.py --all --all-variants --output-dir dist/ # all 6
|
|||||||
python scripts/generate_pack.py --all --all-variants --verify-packs --output-dir dist/
|
python scripts/generate_pack.py --all --all-variants --verify-packs --output-dir dist/
|
||||||
|
|
||||||
# Listing and integrity
|
# Listing and integrity
|
||||||
python scripts/generate_pack.py --list # list available platforms
|
python scripts/generate_pack.py --list-platforms # list available platforms
|
||||||
python scripts/generate_pack.py --list-emulators
|
python scripts/generate_pack.py --list-emulators
|
||||||
python scripts/generate_pack.py --list-systems
|
python scripts/generate_pack.py --list-systems
|
||||||
python scripts/generate_pack.py --platform retroarch --list-targets
|
python scripts/generate_pack.py --platform retroarch --list-targets
|
||||||
|
|||||||
Reference in new issue
Block a user