chore: merge origin/main

This commit is contained in:
Abdessamad Derraz committed 2026-11-06 16:42:16 +01:00
commit 84cffe3f8d
270 files changed
+2107 -70

No files matched your search

+3
View File
@@ -116,6 +116,9 @@ def main() -> int:
parser.add_argument("--json", action="store_true", help="print the list as JSON")
parser.add_argument("--strict", action="store_true", help="exit 1 while a file is listed")
args = parser.parse_args()
from common import work_from_repo_root # noqa: PLC0415
work_from_repo_root(args, parser, ["db", "emulators_dir", "platforms_dir", "install_dir"])
rows = bundled_only(
load_database(args.db), load_emulator_profiles(args.emulators_dir),
+6 -3
View File
@@ -83,14 +83,17 @@ from collections.abc import Callable, Iterator, Sequence
from dataclasses import dataclass, field
from pathlib import Path
import yaml
sys.path.insert(0, str(Path(__file__).resolve().parent))
# First: core_probe stops an interpreter older than 3.9 before anything else
# fails on it.
import core_probe
import yaml
import autorun
import campaign_profiles
import campaign_rules as rules
import core_probe
import measurement_records as mr
ROOT = Path(__file__).resolve().parent.parent
+101 -15
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
from collections.abc import Callable, Iterable
from collections import Counter
import argparse
import filecmp
import hashlib
import functools
@@ -19,6 +20,16 @@ import signal
import sys
import zipfile
from pathlib import Path
from typing import Any, TextIO
# Every build script imports this module. verify.py, slots.py, validate_pr.py
# and cross_reference.py started under 3.8 and stopped halfway on a
# str.removeprefix AttributeError, which named no version.
if sys.version_info < (3, 9):
sys.exit(
f"{Path(sys.argv[0]).name} needs Python 3.9 or newer, "
f"this is {sys.version.split()[0]}"
)
try:
@@ -53,6 +64,33 @@ class DatabaseUnreadable(InputMissing):
"""The database a command was pointed at does not exist or is not JSON."""
class SourceUnreadable(ValueError):
"""A profile or platform file that is not valid YAML. Not an InputMissing:
the callers that skip a platform whose file is absent must not skip one
whose file is damaged. run_cli prints the message on one line."""
def source_yaml(stream: TextIO) -> Any:
"""Parse one of our YAML sources, naming the file a syntax error is in.
Every profile is loaded whatever the command, so one misindented key in
an unrelated profile stopped `verify.py --emulator handy` on a 37-line
traceback, the faulty file named only near its end.
"""
if yaml is None:
return yaml_load(stream)
try:
return yaml_load(stream)
except yaml.YAMLError as exc:
mark = getattr(exc, "problem_mark", None)
where = f"{stream.name}:{mark.line + 1}" if mark else stream.name
problem = getattr(exc, "problem", None) or str(exc).splitlines()[0]
raise SourceUnreadable(
f"{where}: {problem}. Check it with: "
"python scripts/validate_schemas.py --source-only"
) from None
def load_database(db_path: str) -> dict:
"""Load database.json and return parsed dict.
@@ -64,6 +102,18 @@ def load_database(db_path: str) -> dict:
with open(db_path) as f:
return json.load(f)
except FileNotFoundError:
# A mistyped --db beside the real database was told to rescan the
# whole collection into a second file under the typo.
import difflib # noqa: PLC0415
typed = Path(db_path)
close = difflib.get_close_matches(
typed.name, [p.name for p in typed.parent.glob("*.json")], n=1, cutoff=0.8
)
if close:
raise DatabaseUnreadable(
f"{db_path} not found. Did you mean {typed.with_name(close[0])}?"
) from None
raise DatabaseUnreadable(
f"{db_path} not found. Build it with: python scripts/generate_db.py "
f"--force --bios-dir bios --output {db_path}"
@@ -302,7 +352,7 @@ def _inherited_yaml(platform_name: str, platforms_dir: str) -> tuple[dict, list[
raise FileNotFoundError(f"Platform config not found: {config_file}")
with open(config_file) as f:
config = yaml_load(f) or {}
config = source_yaml(f) or {}
if "inherits" not in config:
return config, [platform_name]
@@ -348,7 +398,7 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
reg_real = os.path.realpath(registry_path)
if reg_real not in _shared_yml_cache:
with open(registry_path) as f:
_shared_yml_cache[reg_real] = yaml_load(f) or {}
_shared_yml_cache[reg_real] = source_yaml(f) or {}
reg = _shared_yml_cache[reg_real]
# The platform's own entry first, then its parents' for what it does
# not say: RetroPie's status, scraper and install are its own, its
@@ -387,7 +437,7 @@ def load_platform_config(platform_name: str, platforms_dir: str = "platforms") -
shared_real = os.path.realpath(shared_path)
if shared_real not in _shared_yml_cache:
with open(shared_path) as f:
_shared_yml_cache[shared_real] = yaml_load(f) or {}
_shared_yml_cache[shared_real] = source_yaml(f) or {}
shared = _shared_yml_cache[shared_real]
shared_groups = shared.get("shared_groups", {})
for system in config.get("systems", {}).values():
@@ -494,7 +544,7 @@ def load_data_dir_registry(platforms_dir: str = "platforms") -> dict:
if not os.path.exists(registry_path):
return {}
with open(registry_path) as f:
data = yaml_load(f) or {}
data = source_yaml(f) or {}
return data.get("data_directories", {})
@@ -504,7 +554,7 @@ def load_platform_registry(platforms_dir: str = "platforms") -> dict:
if not os.path.exists(registry_path):
return {}
with open(registry_path) as f:
return (yaml_load(f) or {}).get("platforms", {})
return (source_yaml(f) or {}).get("platforms", {})
def list_registered_platforms(platforms_dir: str = "platforms") -> list[str]:
@@ -521,7 +571,7 @@ def list_registered_platforms(platforms_dir: str = "platforms") -> list[str]:
if not os.path.exists(registry_path):
raise InputMissing(f"no platform registry at {registry_path}")
with open(registry_path) as f:
registry = yaml_load(f) or {}
registry = source_yaml(f) or {}
platforms = []
for name, meta in sorted(registry.get("platforms", {}).items()):
config_path = os.path.join(platforms_dir, meta.get("config", f"{name}.yml"))
@@ -621,6 +671,31 @@ def suggest_names(
return f"{listing_flag} lists all {total}."
def work_from_repo_root(
args: argparse.Namespace, parser: argparse.ArgumentParser, path_flags: list[str]
) -> None:
"""Run from the repository root, whichever directory the command started in.
database.json records every file from the root and the data registry
names its caches the same way, so a run from scripts/ ended on a
FileNotFoundError traceback. A path the caller typed is read from where
they typed it, a default from the root.
"""
root = Path(__file__).resolve().parent.parent
if Path.cwd().resolve() == root:
return
for flag in path_flags:
value = getattr(args, flag, None)
if value and value != parser.get_default(flag):
# A Path stays a Path for the code that joins onto it.
setattr(args, flag, type(value)(os.path.abspath(value)))
os.chdir(root)
if getattr(args, "platform", None):
# Read before the move, the registry was not there to map
# "RetroArch" to its slug.
args.platform = platform_arg(args.platform)
def platform_arg(value: str) -> str:
"""--platform as typed, mapped to the registry's slug when it names one.
@@ -669,7 +744,7 @@ def _target_overrides(platform_name: str, targets_dir: str) -> dict:
if not os.path.exists(overrides_file):
return {}
with open(overrides_file) as f:
all_overrides = yaml_load(f) or {}
all_overrides = source_yaml(f) or {}
return all_overrides.get(platform_name, {}).get("targets", {})
@@ -690,7 +765,7 @@ def platform_files_target(platform: str, target: str, platforms_dir: str = "plat
if not os.path.exists(target_file):
return False
with open(target_file) as f:
targets = (yaml_load(f) or {}).get("targets", {})
targets = (source_yaml(f) or {}).get("targets", {})
return target in _target_alias_index(targets, _target_overrides(platform, targets_dir))
@@ -710,7 +785,7 @@ def canonical_target_name(
if not os.path.exists(target_file):
continue
with open(target_file) as f:
targets = (yaml_load(f) or {}).get("targets", {})
targets = (source_yaml(f) or {}).get("targets", {})
canonical = _target_alias_index(
targets, _target_overrides(platform_name, targets_dir)
).get(target)
@@ -764,7 +839,7 @@ def load_target_config(
f"(platforms with targets: {', '.join(with_targets)})"
)
with open(target_file) as f:
data = yaml_load(f) or {}
data = source_yaml(f) or {}
targets = data.get("targets", {})
overrides = _target_overrides(platform_name, targets_dir)
@@ -822,7 +897,7 @@ def list_available_targets(
if not os.path.exists(target_file):
return []
with open(target_file) as f:
data = yaml_load(f) or {}
data = source_yaml(f) or {}
# The count --target filters on: overrides applied, as load_target_config
# applies them. The scraped list counted Batocera's machine variants and
@@ -2235,7 +2310,7 @@ def load_emulator_profiles(
raise InputMissing(f"no emulator profile directory at {emulators_dir}")
for f in source_yaml_paths(emu_path):
with open(f) as fh:
profile = yaml_load(fh) or {}
profile = source_yaml(fh) or {}
if "emulator" not in profile:
continue
_normalize_hash_fields(profile)
@@ -2365,7 +2440,7 @@ def group_identical_platforms(
try:
raw_path = os.path.join(platforms_dir, f"{platform}.yml")
with open(raw_path) as f:
raw = yaml_load(f) or {}
raw = source_yaml(f) or {}
inherits[platform] = "inherits" in raw
config = load_platform_config(platform, platforms_dir)
except FileNotFoundError:
@@ -3548,6 +3623,9 @@ def build_target_cores_cache(
raise
notes.append(f"INFO: Skipping {p}: no hardware targets")
skip.append(p)
except SourceUnreadable:
# A damaged targets file is not a platform without the target.
raise
except ValueError as e:
if is_all:
notes.append(f"INFO: Skipping {p}: {e}")
@@ -3575,7 +3653,7 @@ def _unknown_target_message(target: str, platforms: list[str], platforms_dir: st
if not os.path.exists(target_file):
continue
with open(target_file) as f:
targets = (yaml_load(f) or {}).get("targets", {})
targets = (source_yaml(f) or {}).get("targets", {})
overrides = _target_overrides(platform, targets_dir)
for name in _target_alias_index(targets, overrides):
owners.setdefault(name, []).append(platform)
@@ -3670,10 +3748,18 @@ def run_cli(main: Callable[[], int | None]) -> None:
for name in ("SIGTERM", "SIGHUP"):
if hasattr(signal, name):
signal.signal(getattr(signal, name), _exit_on_signal)
# Redirected, stdout is block-buffered and stderr is not: the --strict
# verdict landed on the first line of the log, before the report it
# closes. Line buffering keeps the two in the order they were written.
# A stream a caller swapped in (a test's StringIO) has nothing to set.
reconfigure = getattr(sys.stdout, "reconfigure", None)
if reconfigure is not None:
reconfigure(line_buffering=True)
try:
code = main()
sys.stdout.flush()
except (InputMissing, ArtifactLockBusy, ArtifactDirectoryMissing) as exc:
except (InputMissing, SourceUnreadable, ArtifactLockBusy,
ArtifactDirectoryMissing) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
sys.exit(1)
except BrokenPipeError:
+8
View File
@@ -40,6 +40,14 @@ from collections.abc import Iterable, Iterator, Sequence
from dataclasses import dataclass, field
from pathlib import Path
# The probe tools do not import common.py, which states the 3.9 floor for the
# other scripts. Under 3.8 they stopped on a str.removesuffix AttributeError.
if sys.version_info < (3, 9):
sys.exit(
f"{Path(sys.argv[0]).name} needs Python 3.9 or newer, "
f"this is {sys.version.split()[0]}"
)
ROOT = Path(__file__).resolve().parent.parent
MIN_BENCH_REPETITIONS = 5
DEFAULT_WORK_DIR = ROOT / "tmp" / "core_probe"
+3
View File
@@ -1188,6 +1188,9 @@ def main() -> None:
"--list-emulators", action="store_true", help="List available emulators"
)
args = parser.parse_args()
from common import work_from_repo_root # noqa: PLC0415
work_from_repo_root(args, parser, ["emulators_dir", "platforms_dir", "db"])
if args.list_emulators:
ignored = [flag for flag, on in (
("--emulator", args.emulator), ("--platform", args.platform),
+3
View File
@@ -247,6 +247,9 @@ def main() -> int:
help="profiles the rules are checked against (default emulators)")
parser.add_argument("--json", action="store_true", help="print the summary as JSON")
args = parser.parse_args()
from common import work_from_repo_root # noqa: PLC0415
work_from_repo_root(args, parser, ["regions", "extras", "headers", "emulators_dir"])
regions = load_yaml(args.regions)
extras = load_yaml(args.extras)
+17 -7
View File
@@ -43,6 +43,7 @@ from common import (
require_directory,
target_listing_line,
suggest_names,
system_index,
require_platform,
load_target_config,
archive_entry,
@@ -165,7 +166,9 @@ class PackRecord:
self.sources: dict[str, str] = {}
def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> None:
def _add_pack_member(
zf: zipfile.ZipFile | PackRecord, source_path: str, arcname: str
) -> None:
"""Add a file to the pack with metadata that does not depend on the build.
ZipFile.write copies the source file's mtime into the member. For a file
@@ -198,7 +201,9 @@ def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> Non
shutil.copyfileobj(source, target, 1024 * 1024)
def _write_generated_member(zf: zipfile.ZipFile, arcname: str, text: str) -> None:
def _write_generated_member(
zf: zipfile.ZipFile | PackRecord, arcname: str, text: str
) -> None:
"""Write a pack member the builder composes rather than copies.
READMEs, instruction notes and the manifest are produced at build time,
@@ -594,7 +599,7 @@ def _data_directory_members(
def _pack_data_directories(
zf: zipfile.ZipFile,
zf: zipfile.ZipFile | PackRecord,
pack_systems: dict,
data_registry: dict | None,
platform_name: str,
@@ -1496,7 +1501,9 @@ def generate_pack(
def _extract_zip_to_archive(
source_zip: str, dest_prefix: str, target_zf: zipfile.ZipFile
source_zip: str,
dest_prefix: str,
target_zf: zipfile.ZipFile | PackRecord,
) -> None:
"""Extract contents of a source ZIP into target ZIP under dest_prefix."""
with zipfile.ZipFile(source_zip, "r") as src:
@@ -1527,7 +1534,7 @@ def _extract_zip_to_archive(
def _add_zip_to_pack(
source_zip: str,
dest_path: str,
target_zf: zipfile.ZipFile,
target_zf: zipfile.ZipFile | PackRecord,
file_entry: dict | None = None,
) -> None:
"""Write a source ZIP into the pack, byte for byte.
@@ -1575,7 +1582,7 @@ class DamagedArchiveError(OSError):
def _pack_data_tree(
zf: zipfile.ZipFile,
zf: zipfile.ZipFile | PackRecord,
local_cache: str,
dd_dest: str,
seen_destinations: set[str],
@@ -2088,7 +2095,10 @@ def generate_system_pack(
f"No emulators found for system(s): {', '.join(system_ids)}",
file=sys.stderr,
)
print(suggest_names(system_ids, sorted(all_systems), "--list-systems"),
# --list-systems merges spellings that differ in case or separator:
# the hint counted 579 spellings for the 532 lines it prints.
print(suggest_names(system_ids, sorted(all_systems), "--list-systems",
listed=len(system_index(profiles))),
file=sys.stderr)
return None
+1 -1
View File
@@ -14,7 +14,6 @@ Usage:
from __future__ import annotations
from siterender import SECTION_LABELS, WIKI_SRC_DIR
import argparse
import csv
import hashlib
@@ -64,6 +63,7 @@ from common import (
unique_emulator_profiles,
yaml_load,
)
from siterender import SECTION_LABELS, WIKI_SRC_DIR
from artifacts import ArtifactLockBusy, artifact_lock
from artifacts import json_text as _json_text
from cross_reference import FileConsumers, entry_target, held_copy
+3
View File
@@ -146,6 +146,9 @@ def main() -> int:
"--strict", action="store_true", help="exit non-zero when any is found"
)
args = parser.parse_args()
from common import work_from_repo_root # noqa: PLC0415
work_from_repo_root(args, parser, ["bios_dir"])
from common import require_directory # noqa: PLC0415
require_directory(args.bios_dir)
+4 -1
View File
@@ -26,7 +26,10 @@ def list_platforms() -> list[str]:
def main() -> None:
parser = argparse.ArgumentParser(allow_abbrev=False, description="List available platforms")
parser.parse_args()
args = parser.parse_args()
from common import work_from_repo_root # noqa: PLC0415
work_from_repo_root(args, parser, [])
platforms = list_platforms()
+3
View File
@@ -384,6 +384,9 @@ def main() -> int:
"(the pipeline writes derivable.json)",
)
args = parser.parse_args()
from common import work_from_repo_root # noqa: PLC0415
work_from_repo_root(args, parser, ["db", "provenance_dir", "emulators_dir", "write_derivable"])
if args.write_derivable and not args.derive:
parser.error("--write-derivable records what --derive finds: pass both")
+3
View File
@@ -201,6 +201,9 @@ def main() -> int:
)
parser.add_argument("--json", action="store_true", help="Machine-readable output")
args = parser.parse_args()
from common import work_from_repo_root # noqa: PLC0415
work_from_repo_root(args, parser, ["db", "emulators_dir"])
db = load_database(args.db)
profiles = load_emulator_profiles(args.emulators_dir)
+8
View File
@@ -36,6 +36,14 @@ import urllib.request
from collections.abc import Callable, Iterable
from pathlib import Path
# The probe tools do not import common.py, which states the 3.9 floor for the
# other scripts. Under 3.8 they stopped on a str.removesuffix AttributeError.
if sys.version_info < (3, 9):
sys.exit(
f"{Path(sys.argv[0]).name} needs Python 3.9 or newer, "
f"this is {sys.version.split()[0]}"
)
sys.path.insert(0, str(Path(__file__).resolve().parent))
import yaml # noqa: E402
+3
View File
@@ -1037,6 +1037,9 @@ def main() -> int:
"without it the run reports them and exits 0",
)
args = parser.parse_args()
from common import work_from_repo_root # noqa: PLC0415
work_from_repo_root(args, parser, ["db", "platforms_dir", "emulators_dir"])
import contextlib
+41 -38
View File
@@ -41,8 +41,10 @@ from common import (
_foreign_tree,
_owner_tokens,
platform_arg,
work_from_repo_root,
target_listing_line,
suggest_names,
system_index,
require_platform,
archive_entry,
archive_is_optional,
@@ -130,29 +132,6 @@ DEFAULT_EMULATORS_DIR = "emulators"
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def work_from_repo_root(
args: argparse.Namespace, parser: argparse.ArgumentParser, path_flags: list[str]
) -> None:
"""Run from the repository root, whichever directory the command started in.
database.json records every file from the root and the data registry
names its caches the same way, so a run from scripts/ ended on a
FileNotFoundError traceback. A path the caller typed is read from where
they typed it, a default from the root.
"""
if os.path.realpath(os.getcwd()) == os.path.realpath(REPO_ROOT):
return
for flag in path_flags:
value = getattr(args, flag, None)
if value and value != parser.get_default(flag):
setattr(args, flag, os.path.abspath(value))
os.chdir(REPO_ROOT)
if getattr(args, "platform", None):
# Read before the move, the registry was not there to map
# "RetroArch" to its slug.
args.platform = platform_arg(args.platform)
# Status model: aligned with Batocera BiosStatus (batocera-systems:967-969)
@@ -1813,7 +1792,8 @@ def _print_undeclared_entry(
ref_part = f" [{u['source_ref']}]" if u.get("source_ref") else ""
print(f" validates {checks_label}{ref_part}")
else:
print(f" [{'+'.join(checks)}]")
# A bare "[size]" under the entry said nothing to a reader.
print(f" validates {'+'.join(checks)}")
def _split_undeclared(undeclared: list[dict]) -> tuple[list[dict], list[dict], list[dict]]:
@@ -1925,10 +1905,15 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
u for u in bundled
if u["required"] and u.get("category", "bios") != "game_data"
]
names = ", ".join(sorted({u["name"] for u in shown}))
named = sorted({u["name"] for u in shown})
# The required firmware alone read as the whole count: 14 names
# for 154 entries, nothing saying the rest existed.
hidden = len({u["name"] for u in bundled}) - len(named)
more = f"{hidden} more, " if named else ""
items = named + ([f"{more}--verbose lists them"] if hidden else [])
print(
f" Bundled with their emulator, not collected: {len(bundled)}"
+ (f" ({names})" if names else "")
+ (f" ({', '.join(items)})" if items else "")
)
if unplaced:
print(f" Core files held but not packed: {len(unplaced)} "
@@ -2619,7 +2604,10 @@ def verify_system(
f"No emulators found for system(s): {', '.join(system_ids)}",
file=sys.stderr,
)
print(suggest_names(system_ids, sorted(all_systems), "--list-systems"),
# --list-systems merges spellings that differ in case or separator:
# the hint counted 579 spellings for the 532 lines it prints.
print(suggest_names(system_ids, sorted(all_systems), "--list-systems",
listed=len(system_index(profiles))),
file=sys.stderr)
sys.exit(1)
@@ -2829,7 +2817,7 @@ def _main(held: contextlib.ExitStack) -> None:
parser = argparse.ArgumentParser(allow_abbrev=False, description="Platform-native BIOS verification")
parser.add_argument("--platform", "-p", type=platform_arg, help="Platform name")
parser.add_argument(
"--all", action="store_true", help="Verify all active platforms"
"--all", action="store_true", help="Verify every registered platform"
)
parser.add_argument(
"--emulator", "-e", help="Emulator profile name(s), comma-separated"
@@ -3145,6 +3133,13 @@ def _overall_summary(group_results: list[tuple[dict, list[str]]]) -> str:
named = [f"{name} {count}" for name, count in counts.items() if count]
return f" ({', '.join(named)})" if named else ""
def on(groups: tuple[str, ...]) -> str:
"""The platforms a file is missing on, by the shorter side."""
if len(group_results) < 2 or len(groups) * 2 <= len(group_results):
return ", ".join(groups)
others = [" / ".join(g) for _, g in group_results if " / ".join(g) not in groups]
return "every platform" + (f" but {', '.join(others)}" if others else "")
parts = []
for severity, label in ((Severity.CRITICAL, "critical"), (Severity.WARNING, "warning"),
(Severity.INFO, "info")):
@@ -3163,18 +3158,26 @@ def _overall_summary(group_results: list[tuple[dict, list[str]]]) -> str:
parts.append(f"{total} {'discrepancy' if total == 1 else 'discrepancies'}{where(counts)}")
lines = [f"{platforms} platforms: {', '.join(parts)}"]
missing = {
" / ".join(group): sorted({
# Grouped by file, not by platform: sfiii2a.zip missing everywhere was
# counted and printed once per platform, 24 files for 10 distinct ones.
holders: dict[str, list[str]] = {}
for r, group in group_results:
for name in sorted({
u["name"] for u in _core_required_missing(_split_undeclared(
r.get("undeclared_files", []))[1])
})
for r, group in group_results
}
total = sum(len(names) for names in missing.values())
if total:
detail = "; ".join(f"{group}: {', '.join(names)}"
for group, names in missing.items() if names)
lines.append(f"{total} core file(s) a core cannot run without are missing ({detail})")
}):
holders.setdefault(name, []).append(" / ".join(group))
if holders:
by_platforms: dict[tuple[str, ...], list[str]] = {}
for name, groups in sorted(holders.items()):
by_platforms.setdefault(tuple(groups), []).append(name)
detail = "; ".join(
f"{', '.join(names)} ({on(groups)})"
for groups, names in sorted(by_platforms.items(),
key=lambda item: (-len(item[0]), item[1]))
)
lines.append(f"{len(holders)} core file(s) a core cannot run without are "
f"missing: {detail}")
for category, label in _NOT_FIRMWARE.items():
counts = {
" / ".join(group): len((r.get("missing_not_firmware") or {}).get(category, []))