chore: merge the launcher evidence rule

This commit is contained in:
Abdessamad Derraz committed 2026-11-11 11:15:53 +01:00
commit 5c1d810086
50 files changed
+2852 -475

No files matched your search

+7 -6
View File
@@ -3189,11 +3189,12 @@ def standalone_bindings_for(
``systems`` are those the entry is launched for (entry_launch_systems).
"""
declared = profile.get("standalone_directories") or []
pools = profile.get("standalone_pools") or {}
from launchers import directory_handles, pool_folders, standalone_bindings # noqa: PLC0415
declared = directory_handles(profile)
pools = pool_folders(profile)
if not declared and not pools:
return {}
from launchers import standalone_bindings # noqa: PLC0415
names = {emu_name, *(str(c) for c in profile.get("cores") or [])}
layout = [
@@ -3298,9 +3299,9 @@ def platform_layouts(
# The layout folders a search setting stands for (standalone_pools):
# bound, the launcher passes the setting and the default is not read.
pool_folders = {
str(v).strip("/") for v in (profile.get("standalone_pools") or {}).values()
} - {""}
from launchers import pool_folders as _pool_folders # noqa: PLC0415
pool_folders = {v.strip("/") for v in _pool_folders(profile).values()} - {""}
def passed_setting(entry: dict) -> bool:
layout = str(standalone_layout_path(entry, profile) or "")
+45 -2
View File
@@ -56,9 +56,11 @@ from common import (
from hashing import parse_hash_list
from launchers import (
LauncherLayer,
bound_launchers,
catalogued_model_files,
declared_destinations,
held_record,
launcher_unproven,
name_settings,
set_by,
setting_readers,
@@ -1095,6 +1097,8 @@ def _settle_launcher_places(
# The --json key of the launcher reads category, beside the emulator keys.
LAUNCHER_READS_KEY = "_launcher_reads_not_held"
# The --json key of the launcher records a config file does not prove.
LAUNCHER_UNPROVEN_KEY = "_launcher_unproven"
# A package record names no file: it records what a package builds.
_NAMELESS_KINDS = frozenset({"package"})
@@ -1238,7 +1242,9 @@ def launcher_reads_not_held(
rows: list[dict] = []
for platform, config in configs.items():
by_key: dict[str, dict] = {}
for record in config.get("launchers") or []:
# A record resting on a config file it cannot prove reads nothing
# yet: it is reported apart (launcher_unproven).
for record in bound_launchers(config, profiles):
names = launcher_file_names(record)
if not names:
continue
@@ -1331,6 +1337,25 @@ def launcher_reads_not_held(
return rows
def print_launcher_unproven(rows: list[dict]) -> None:
"""The launcher records resting on a config file the code does not prove.
Neither a pack, a manifest nor a severity reads them: each is listed
with the links it lacks.
"""
if not rows:
return
platforms = sorted({r["platform"] for r in rows})
noun = "platform" if len(platforms) == 1 else "platforms"
print(f"\nLauncher records unproven: {len(rows)} across {len(platforms)} {noun}")
for platform in platforms:
print(f"\n{platform}")
for row in (r for r in rows if r["platform"] == platform):
print(f" [{row['index']}] {row['emulator']} {row['kind']}: {row['passes_to']}")
for link in row["missing"]:
print(f" missing {link}")
def print_launcher_reads(rows: list[dict]) -> None:
"""The launcher reads gap category, after the profile gaps.
@@ -1649,8 +1674,25 @@ def main() -> None:
db, all_profiles, registry, emulator or "",
)
unproven = [
row for row in launcher_unproven(
{
name: load_platform_config(name, args.platforms_dir)
for name in (selected or list_registered_platforms(args.platforms_dir))
},
all_profiles,
)
if not emulator or any(
emulator in LauncherLayer._profiles_named(part, all_profiles, set(all_profiles))
for part in row["emulator"].split(", ")
)
]
if args.json:
print(json.dumps({**report, LAUNCHER_READS_KEY: launcher_reads}, indent=2))
print(json.dumps(
{**report, LAUNCHER_READS_KEY: launcher_reads, LAUNCHER_UNPROVEN_KEY: unproven},
indent=2,
))
else:
if emulator:
scope = emulator
@@ -1660,6 +1702,7 @@ def main() -> None:
scope = "all emulators"
print_report(report, scope, show_clean=bool(args.emulator))
print_launcher_reads(launcher_reads)
print_launcher_unproven(unproven)
if __name__ == "__main__":
+30 -2
View File
@@ -3647,13 +3647,15 @@ def _served_line(f: dict, declared_sha1: str = "") -> str:
def _launcher_refs(config: dict) -> dict[str, list[str]]:
"""Entry name, casefolded -> each launcher record that opens it, rendered."""
from launchers import _names as launcher_names
from launchers import bound_launchers
from sitesources import launcher_ref_markdown, list_source
# Every scraper reads its launchers in the repository and at the revision
# of the list it scrapes, so `source` pins each source_ref.
# of the list it scrapes, so `source` pins each source_ref. A record a
# config file does not prove opens nothing: it is listed apart.
source = list_source(str(config.get("source") or ""))
found: dict[str, list[str]] = {}
for record in config.get("launchers") or []:
for record in bound_launchers(config):
if not isinstance(record, dict) or not record.get("source_ref"):
continue
text = (f"{_escape_tags(str(record.get('emulator', '')))} at "
@@ -3665,6 +3667,31 @@ def _launcher_refs(config: dict) -> dict[str, list[str]]:
return found
UNPROVEN_TITLE = "Launcher records a config does not prove"
def _launcher_unproven_section(config: dict) -> list[str]:
"""The platform's launcher records resting on a config the code does not prove.
A config file is proof only when the code installing it and the code
reading it are shown: these records place no file.
"""
from launchers import launcher_unproven
from sitesources import launcher_ref_markdown, list_source
rows = launcher_unproven({str(config.get("_key") or ""): config})
if not rows:
return []
source = list_source(str(config.get("source") or ""))
body = [
f"- {_escape_tags(row['emulator'])}, `{_escape_tags(row['passes_to'] or row['kind'])}` "
f"at {launcher_ref_markdown(source, row['source_ref'])}: "
f"{_escape_tags('; '.join(row['missing']))}"
for row in rows
]
return ["", *markup.details("note", f"{UNPROVEN_TITLE} ({len(rows)})", body)]
# The hash a row shows first, when the list does not say which it checks.
_ROW_HASHES = ("sha1", "md5", "sha256", "crc32")
# An untested entry whose reason is a hash the bytes do not match.
@@ -4799,6 +4826,7 @@ def generate_platform_page(
_render_platform_systems(by_system, config_files, emulator_files,
(name, display), moved, config)
)
lines.extend(_launcher_unproven_section(config))
lines.extend(["", _date_line()])
return "\n".join(lines) + "\n"
+314 -5
View File
@@ -254,11 +254,318 @@ def _names(record: dict) -> list[str]:
return names
def launcher_places(config: dict) -> list[LauncherPlace]:
# How a record's passes_to names what carries the file when it names no
# config file: an option on the command line, or a libretro core option.
COMMAND_LINE = "command line"
CORE_OPTIONS = "core options"
_TAIL_WORDS = re.compile(r"\s+(?:unset|left empty)$")
_KEY_TOKEN = re.compile(r"^-{0,2}[\w.$()/\\]+$")
def config_target(record: dict) -> tuple[str, list[str]]:
"""The config a record writes its file to, and the keys it names there.
passes_to spells them as describe_keys does: "settings.ini [BIOS]
SearchDirectory, [BIOS] PathNTSCU, PathNTSCJ unset", "command line
-hard", "core option pcsx2_bios", "hatarib_tos = hatarib/<value>". A
part that reads as prose ("only if the file exists") names no key. When
passes_to names no file ("romlist coco"), the file is the one the record
installs and the whole of passes_to is the key.
"""
# A parenthesis says how the file is passed, unless a config variable
# opens it: $(FILE_PATH) is amiberry's.
text = re.sub(r"(?<!\$)\([^()]*\)", "", str(record.get("passes_to") or "")).strip()
head, _, rest = text.partition(" ")
if text.startswith(COMMAND_LINE + " "):
flag = text[len(COMMAND_LINE):].split()[0]
return COMMAND_LINE, [flag]
if text.startswith("core option "):
return CORE_OPTIONS, [text[len("core option "):].split()[0]]
if "." not in head or head.startswith("<"):
# The file is the one the record installs, named by its path or,
# when a variable spells the path, by the config file it cites.
from scraper.shell_launchers import config_ref, is_feature_ref, ref_path # noqa: PLC0415
installed = str((record.get("installs") or {}).get("path") or "").rsplit("/", 1)[-1]
cited = [
ref_path(part).rsplit("/", 1)[-1]
for part in config_ref(str(record.get("source_ref") or ""))
if not is_feature_ref(part)
]
if "." in installed or (installed and not cited):
return installed, [text]
if installed and cited:
return cited[0], [text]
if record.get("build") == "libretro" and record.get("kind") == "core_option":
return CORE_OPTIONS, [head]
return "", []
keys: list[str] = []
section = ""
for piece in re.split(r",\s*", rest):
piece = _TAIL_WORDS.sub("", piece.strip())
# A section opens anywhere in a piece: "BIOS9Path [DSi] FirmwarePath".
for chunk in re.split(r"(\[[^\]]+\])", piece):
chunk = chunk.strip()
if not chunk:
continue
if chunk.startswith("[") and chunk.endswith("]"):
section = chunk
continue
# "A = B = <value>" sets every key before the value.
names = [p.strip() for p in re.split(r"\s*=\s*", chunk)]
if len(names) > 1:
names = names[:-1]
for name in names:
if not _KEY_TOKEN.match(name):
continue
key = f"{section} {name}" if section else name
if key not in keys:
keys.append(key)
return head, keys
def _read_keys(read: dict) -> dict[str, str]:
"""Each key a config_reads entry says the build reads, to its source_ref."""
found: dict[str, str] = {}
for entry in read.get("keys") or []:
spelled = entry.get("key")
for key in spelled if isinstance(spelled, list) else [spelled]:
if key:
found[str(key)] = str(entry.get("source_ref") or "")
return found
def _handle_refs(profile: dict) -> dict[str, str]:
"""The handles a profile names with the code that reads them."""
found: dict[str, str] = {}
for item in profile.get("standalone_directories") or []:
if isinstance(item, dict) and item.get("source_ref"):
handle = str(item.get("handle") or "").partition(" = ")[0]
found[handle] = str(item["source_ref"])
for handle, value in (profile.get("standalone_pools") or {}).items():
if isinstance(value, dict) and value.get("source_ref"):
found[str(handle)] = str(value["source_ref"])
return found
def directory_handles(profile: dict) -> list[str]:
"""standalone_directories as handle strings, a {handle, source_ref} item
by its handle."""
return [
str(item.get("handle") or "") if isinstance(item, dict) else str(item)
for item in profile.get("standalone_directories") or []
]
def pool_folders(profile: dict) -> dict[str, str]:
"""standalone_pools as handle to subfolder, a {folder, source_ref} value
by its folder."""
return {
str(handle): str(value.get("folder") or "") if isinstance(value, dict) else str(value)
for handle, value in (profile.get("standalone_pools") or {}).items()
}
def _reads_apply(read: dict, profile: dict, platform: str) -> bool:
"""Whether a config_reads entry describes the build ``platform`` ships.
An entry naming platforms is read at their pin_build revision. One that
names none is read at the profile's pin, the build of every platform
with no pin_build of its own or one pinning the same commit.
"""
named = [str(p) for p in read.get("platforms") or []]
if named:
return platform in named
pin = str(profile.get("source_commit") or "")
builds = [b for b in profile.get("pin_build") or [] if b.get("platform") == platform]
return all(pin and str(b.get("value") or "").startswith(pin[:12]) for b in builds)
def _file_matches(read_file: str, file: str, record: dict) -> bool:
"""Whether a config_reads file is the one a record writes.
The entry names the file as the build finds it: a name, a glob ("*.uae",
any file the launcher passes), a path, or the flag the launcher passes
it with (amiberry's -f).
"""
from fnmatch import fnmatch # noqa: PLC0415
spelled = read_file.rsplit("/", 1)[-1].casefold()
if file and fnmatch(file.casefold(), spelled):
return True
installed = str((record.get("installs") or {}).get("path") or "").rsplit("/", 1)[-1]
if installed and fnmatch(installed.casefold(), spelled):
return True
return read_file.startswith("-") and read_file in str(record.get("passes_to") or "").split()
@dataclass(frozen=True)
class ConfigEvidence:
"""Whether a launcher record that rests on a config holds, and why not.
``config`` holds the parts of its source_ref that cite a config, option
or feature file. A record citing none, and choosing among no listed
option values, rests on code alone and holds.
"""
config: tuple[str, ...]
missing: tuple[str, ...]
readers: tuple[str, ...] = ()
@property
def proven(self) -> bool:
return not self.missing
def _named_profiles(record: dict, profiles: Mapping[str, dict], platform: str) -> list[str]:
"""The profiles a record's emulator names, a platform's own build first."""
found = []
for part in (e.strip() for e in str(record.get("emulator", "")).split(",")):
named = [
key for key, profile in profiles.items()
if profile.get("type") not in ("alias", "launcher")
and (key == part or part in {str(c) for c in profile.get("cores") or []})
]
own = [k for k in named if platform in (profiles[k].get("built_by") or [])]
found.extend(own or [k for k in named if not profiles[k].get("built_by")])
return sorted(set(found))
def config_evidence(
record: dict, profiles: Mapping[str, dict], platform: str = ""
) -> ConfigEvidence:
"""What a record citing a config file proves, link by link.
A config file is proof only when the code that installs it and the code
that reads it are shown: (a) the platform installs the file at a path
(``installs``), (b) its launch code was read for what replaces the file
or the key (``overrides``), (c) the emulator loads that file and (d)
reads each key and opens the value (``config_reads`` of its profile, at
the build the platform ships). A feature or option file only lists the
values a menu offers: it goes in ``choices_from``, beside the code that
writes the choice, and such a record needs (c) and (d) too.
"""
from scraper.shell_launchers import config_ref, is_feature_ref # noqa: PLC0415
cited = config_ref(str(record.get("source_ref") or ""))
choices = str(record.get("choices_from") or "")
if not cited and not choices:
return ConfigEvidence((), ())
missing: list[str] = []
features = [part for part in cited if is_feature_ref(part)]
if features:
missing.append("an option list cited as source, not as choices_from")
shipped = [part for part in cited if not is_feature_ref(part)]
if shipped:
installs = record.get("installs") or {}
if not (installs.get("path") and installs.get("source_ref")):
missing.append("installs: the code that puts the config in place")
if "overrides" not in record:
missing.append("overrides: the launch code read for what replaces it")
file, keys = config_target(record)
if not keys:
missing.append("a key the config hands the file to")
readers: list[str] = []
uncovered = list(keys)
loaded = False
for key in _named_profiles(record, profiles, platform) or sorted(profiles):
profile = profiles[key]
handles = _handle_refs(profile)
for read in profile.get("config_reads") or []:
if not _reads_apply(read, profile, platform):
continue
if not _file_matches(str(read.get("file") or ""), file, record):
continue
loaded = True
known = _read_keys(read)
before = len(uncovered)
uncovered = [k for k in uncovered if not (known.get(k) or handles.get(k))]
if len(uncovered) < before and key not in readers:
readers.append(key)
if not loaded:
missing.append(f"config_reads: an emulator that loads {file or 'the config'}")
elif uncovered:
missing.append("config_reads: no read of " + ", ".join(uncovered))
return ConfigEvidence(tuple(cited or [choices]), tuple(missing), tuple(readers))
_DEFAULT_PROFILES: dict[str, dict] | None = None
def _profiles_or_repo(profiles: Mapping[str, dict] | None) -> Mapping[str, dict]:
"""The profiles given, else this repository's, read once."""
global _DEFAULT_PROFILES # noqa: PLW0603
if profiles is not None:
return profiles
if _DEFAULT_PROFILES is None:
from pathlib import Path # noqa: PLC0415
from common import load_emulator_profiles # noqa: PLC0415
_DEFAULT_PROFILES = load_emulator_profiles(
str(Path(__file__).resolve().parent.parent / "emulators")
)
return _DEFAULT_PROFILES
def _cites_config(record: dict) -> bool:
from scraper.shell_launchers import config_ref # noqa: PLC0415
return bool(record.get("choices_from")) or bool(config_ref(str(record.get("source_ref") or "")))
def bound_launchers(config: dict, profiles: Mapping[str, dict] | None = None) -> list[dict]:
"""The launcher records a platform's packs, manifests and reports consume.
A record resting on a config file binds only when config_evidence holds
for it. The repository's profiles are read only when a record cites one.
"""
records = [r for r in config.get("launchers") or [] if isinstance(r, dict)]
if not any(_cites_config(r) for r in records):
return records
known = _profiles_or_repo(profiles)
platform = str(config.get("_key") or "")
return [r for r in records if config_evidence(r, known, platform).proven]
def launcher_unproven(
configs: Mapping[str, dict], profiles: Mapping[str, dict] | None = None
) -> list[dict]:
"""The records resting on a config file whose evidence does not hold.
One row per record, with the config parts it cites and each missing
link. Reported apart: no severity, count or exit code reads it.
"""
rows: list[dict] = []
for platform, config in configs.items():
records = [r for r in config.get("launchers") or [] if isinstance(r, dict)]
if not any(_cites_config(r) for r in records):
continue
known = _profiles_or_repo(profiles)
for index, record in enumerate(records):
evidence = config_evidence(record, known, str(config.get("_key") or platform))
if evidence.proven:
continue
rows.append({
"platform": platform,
"index": index,
"emulator": str(record.get("emulator", "")),
"kind": str(record.get("kind", "")),
"passes_to": str(record.get("passes_to") or ""),
"source_ref": str(record.get("source_ref", "")),
"config": list(evidence.config),
"missing": list(evidence.missing),
})
return rows
def launcher_places(
config: dict, profiles: Mapping[str, dict] | None = None
) -> list[LauncherPlace]:
"""Every place a platform's launchers open a named file from."""
base = str(config.get("base_destination") or "bios")
places: list[LauncherPlace] = []
for record in config.get("launchers") or []:
for record in bound_launchers(config, profiles):
kind = record.get("kind")
if kind not in PLACE_KINDS:
continue
@@ -484,7 +791,9 @@ def _listed_directories(raw: str) -> list[str] | None:
return None
def handed_directories(config: dict) -> list[HandedDirectory]:
def handed_directories(
config: dict, profiles: Mapping[str, dict] | None = None
) -> list[HandedDirectory]:
"""Every whole directory a platform links or passes to a standalone build.
A record naming files (gpsp's gba_bios.bin) or one slot of a directory
@@ -499,7 +808,7 @@ def handed_directories(config: dict) -> list[HandedDirectory]:
natives = native_systems(config)
declared = {d.lower() for d in declared_destinations(config)}
found: list[HandedDirectory] = []
for record in config.get("launchers") or []:
for record in bound_launchers(config, profiles):
if record.get("build") != "standalone" or record.get("kind") not in HANDED_KINDS:
continue
if record.get("names") or record.get("slot"):
@@ -988,7 +1297,7 @@ class LauncherLayer:
db: dict,
declared: Iterable[str],
) -> None:
self.places = launcher_places(config)
self.places = launcher_places(config, profiles)
self.declared = set(declared)
self.declared_folded = {d.lower() for d in self.declared}
self.declared_names = {d.rsplit("/", 1)[-1] for d in self.declared}
+97
View File
@@ -1047,6 +1047,11 @@ class ProfileReport:
new_files_elided: dict[str, int] = field(default_factory=dict)
new_hashes: list[dict] | None = None
detection_skipped: str | None = None
# config_reads refs written for a build a platform ships, each read at
# that build (pin_build) rather than at source_commit: {platform,
# revision, ref, status, detail}. A status other than BUILD_OK is review
# work.
build_refs: list[dict] = field(default_factory=list)
# Skips that say the profile cannot be checked as written: a repository
# that no longer answers, no pin to read, a death the forge contradicts.
@@ -1070,6 +1075,7 @@ class ProfileReport:
+ len(self.orphaned)
+ len(self.stale_prose or ())
+ len(self.moved_repos)
+ sum(1 for ref in self.build_refs if ref["status"] != "BUILD_OK")
)
@@ -1603,6 +1609,81 @@ def moved_repositories(
return moved
def shipped_build_holders(profile: dict) -> dict[int, dict]:
"""The source_ref holders of config_reads entries naming platforms, by id.
Such an entry describes the build those platforms ship, so its refs are
read at their pin_build revision, never at source_commit.
"""
found: dict[int, dict] = {}
for read in profile.get("config_reads") or []:
if not isinstance(read, dict) or not read.get("platforms"):
continue
found[id(read)] = read
for key in read.get("keys") or []:
if isinstance(key, dict):
found[id(key)] = read
return found
def build_revision(profile: dict, platform: str) -> str | None:
"""The emulator revision a platform's pin_build entry names, None when
its value is no revision (a URL, a package name)."""
for build in profile.get("pin_build") or []:
if build.get("platform") != platform:
continue
value = str(build.get("value") or "").strip()
return value if value and re.fullmatch(r"[\w.+-]+", value) else None
return None
def _check_build_refs(
profile: dict,
deferred: list[tuple[Citation, dict]],
views: list[RepoView],
fetch: Callable[[upstream.Repo, str, str], list[str] | None],
) -> list[dict]:
"""Each deferred config_reads ref read at the build of its platforms.
The file must exist at that revision and every cited line lie inside
it. The entry declares no value to anchor on, so this is the check a
ref past the end of its file fails (BUILD_GONE), not a content match.
"""
results: list[dict] = []
for citation, read in deferred:
for platform in read.get("platforms") or []:
revision = str(read.get("revision") or "") or build_revision(profile, str(platform))
row = {"platform": str(platform), "revision": revision or "",
"ref": citation.ref, "field": citation.field}
if revision is None:
results.append({**row, "status": "BUILD_UNREAD",
"detail": "no pin_build revision for this platform"})
continue
problems: list[str] = []
for part in split_source_ref(citation.ref):
role, path = ref_role(part.path)
if " " in path:
# A project word in front (`ROCKNIX patches/x.patch:4`):
# a file of the platform's tree, read with the record.
continue
lines = next(
(found for view in views
if (not role or view.field == role)
and (found := fetch(view.repo, revision, path)) is not None),
None,
)
if lines is None:
problems.append(f"{part.path} absent at {revision[:12]}")
elif (part.end or part.start or 0) > len(lines):
problems.append(f"{part.raw} past the end ({len(lines)} lines)")
results.append({
**row,
"status": "BUILD_GONE" if problems else "BUILD_OK",
"detail": "; ".join(problems),
})
return results
def build_report(
name: str, profile: dict, cache_dir: str, offline: bool = False,
text: str | None = None,
@@ -1621,7 +1702,13 @@ def build_report(
)
modes = {mode for _, mode, _ in declared_repositories(profile) if mode}
refs = []
at_build = shipped_build_holders(profile)
deferred: list[tuple[Citation, dict]] = []
for citation in collect_citations(profile, text):
if citation.kind == "ref" and id(citation.holder) in at_build:
# Written for the build a platform ships: read at pin_build.
deferred.append((citation, at_build[id(citation.holder)]))
continue
if citation.kind == "ref":
entry = citation.entry
display = str(entry.get("name") or "") or citation.field
@@ -2204,6 +2291,8 @@ def build_report(
)
return result
report.build_refs = _check_build_refs(profile, deferred, views, fetch_from)
staged = []
for entry_name, ref, tokens, hashes, ref_parts, citation in refs:
flags = [self_checked(part, tokens) for part in ref_parts]
@@ -2335,6 +2424,14 @@ def format_report(report: ProfileReport, changed_only: bool = False) -> str:
lines.append(f" {len(report.entries)} refs: {summary}")
for line in report.stale_prose or ():
lines.append(f" prose under an older pin: {line}")
for ref in report.build_refs:
if changed_only and ref["status"] == "BUILD_OK":
continue
detail = f": {ref['detail']}" if ref["detail"] else ""
lines.append(
f" {ref['status']} at {ref['platform']} build "
f"{ref['revision'][:12] or '-'} {ref['ref']}{detail}"
)
lines.append(f" {report.needs_review()} to review")
return "\n".join(lines)
+55 -29
View File
@@ -7,7 +7,7 @@ runs the script its standalone_emulators table names (/usr/bin/<name>.sh,
installed from the package). Those scripts, and the configs their packages
install, say which file under /storage/roms/bios is opened: yabasanshiro.sh
tests saturn_bios.bin and passes it with -b, Hatari's default config names
tos.img, the atari800 core is built to read a config naming its OS ROMs.
tos.img.
runemu.py is read with ast, the scripts with shell_launchers' readers. An
extractor whose construct is gone raises instead of returning nothing.
@@ -26,7 +26,9 @@ try:
from .shell_launchers import (
LauncherPatternMissing,
Script,
config_placement,
describe_keys,
evidence_fields,
settings,
source_ref,
)
@@ -35,7 +37,9 @@ except ImportError:
from shell_launchers import ( # type: ignore[no-redef]
LauncherPatternMissing,
Script,
config_placement,
describe_keys,
evidence_fields,
settings,
source_ref,
)
@@ -45,8 +49,8 @@ RUNEMU = "packages/amberelec/sources/scripts/runemu.py"
RETROARCH_CFG = "packages/games/tools/retroarch/sources/retroarch.cfg"
RETRORUN = "packages/games/tools/retrorun/retrorun.sh"
HATARI_CONFIG = "packages/games/emulators/hatarisa/config/conf/Atari-ST-default.cfg"
ATARI800_CONFIG = "packages/amberelec/config/distribution/configs/atari800.cfg"
ATARI800_PATCH = "packages/games/libretro/atari800/patches/atari800-01-add-emuelec-platform.patch"
# The first-boot copy of every package's /usr/config into /storage/.config.
USERCONFIG = "packages/sysutils/systemd/scripts/userconfig-setup"
AMIBERRY_PACKAGE = "packages/games/emulators/amiberry/package.mk"
AMIBERRY_CONFIGS = "packages/games/emulators/amiberry/config/conf"
FREEJ2ME = (
@@ -103,16 +107,35 @@ class Runemu:
return path, line
def _evidence(
read: Read, record: Launcher, target: str, keys: list[str],
start: Script | None = None, source_name: str = "",
) -> Launcher:
"""A config record with the steps that put its file at ``target``: the
first-boot copy of /usr/config, then the start script's own copy."""
setup = Script(USERCONFIG, read(USERCONFIG))
placements = [config_placement(setup, target, required=False, sources=("/usr/config/",))]
if start is not None:
placements.append(
config_placement(start, target, keys, source_name=source_name, required=False)
)
fields = evidence_fields(placements)
record.installs = fields["installs"]
record.kept_if_present = fields["kept_if_present"]
record.overrides = fields["overrides"]
return record
def retroarch(read: Read) -> list[Launcher]:
"""system_directory of the retroarch.cfg the retroarch package installs."""
found = [s for s in settings(read(RETROARCH_CFG)) if s.key == "system_directory"]
if len(found) != 1:
raise LauncherPatternMissing(RETROARCH_CFG, "one system_directory")
return [Launcher(
return [_evidence(read, Launcher(
emulator="retroarch", build="libretro", kind="passes_path", names=[],
search_dir=found[0].value, passes_to="retroarch.cfg system_directory",
fails_when_missing=False, source_ref=source_ref([(RETROARCH_CFG, [found[0].line])]),
)]
), "/storage/.config/retroarch/retroarch.cfg", ["system_directory"])]
def retrorun(read: Read) -> list[Launcher]:
@@ -170,10 +193,12 @@ def hatari(read: Read, runemu: Runemu) -> list[Launcher]:
path, table_line = runemu.script("HATARISA")
script = Script(path, read(path))
name = posixpath.basename(HATARI_CONFIG)
copied, _ = script.find(rf"^\s*cp\s+\S*/{re.escape(name)}\s", f"cp of {name}")
copied, copy = script.find(
rf"^\s*cp\s+\S*/{re.escape(name)}\s+\"?(\$\{{?\w+\}}?)", f"cp of {name}"
)
passed, _ = script.find(r"\shatari\s+--configfile\s", "hatari --configfile")
files, lines = _bios_files(HATARI_CONFIG, read(HATARI_CONFIG))
return [Launcher(
return [_evidence(read, Launcher(
emulator="HATARISA", build="standalone", kind="fixed_name",
names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS,
passes_to=describe_keys(name, tuple(k for k, _n in files)),
@@ -181,24 +206,8 @@ def hatari(read: Read, runemu: Runemu) -> list[Launcher]:
source_ref=source_ref([
(RUNEMU, [table_line]), (path, [copied, passed]), (HATARI_CONFIG, lines),
]),
)]
def atari800(read: Read) -> list[Launcher]:
"""The ROM keys of the config the atari800 core is built to read."""
patch = Script(ATARI800_PATCH, read(ATARI800_PATCH))
line, found = patch.find(r'DEFAULT_CFG_NAME="\\"([^"\\]+)\\""', "DEFAULT_CFG_NAME")
built = found.group(1)
if not built.endswith("/" + ATARI800_CONFIG.split("/config/", 1)[1]):
raise LauncherPatternMissing(ATARI800_PATCH, f"DEFAULT_CFG_NAME naming {ATARI800_CONFIG}")
files, lines = _bios_files(ATARI800_CONFIG, read(ATARI800_CONFIG))
return [Launcher(
emulator="atari800", build="libretro", kind="fixed_name",
names=[n for _k, n in files], case_insensitive=False, search_dir=BIOS,
passes_to=describe_keys(posixpath.basename(ATARI800_CONFIG), tuple(k for k, _n in files)),
fails_when_missing=False,
source_ref=source_ref([(ATARI800_PATCH, [line]), (ATARI800_CONFIG, lines)]),
)]
), script.expand(copy.group(1), copied, set()),
[f"[{s}] {k}" if s else k for (s, k), _n in files], script, name)]
def yabasanshiro(read: Read, runemu: Runemu) -> list[Launcher]:
@@ -279,6 +288,15 @@ def amiberry(read: Read, runemu: Runemu) -> list[Launcher]:
installed = re.sub(r"^\$\{INSTALL\}", "", link.link)
path, table_line = runemu.script("AMIBERRY")
script = Script(path, read(path))
# A WHDLoad archive is booted by --autoload, which builds its own setup
# over the config: the kickstart records hold for the other content.
autoload = [n for n, text in script.code() if re.search(r"amiberry .*--autoload", text)]
if not autoload:
raise script.missing("WHDLoad boot with --autoload")
content = (
"content not booted with --autoload, a WHDLoad .lha or .zip "
f"({posixpath.basename(path)}:{', '.join(map(str, autoload))})"
)
launchers = [Launcher(
emulator="AMIBERRY", build="standalone", kind="links", names=[],
case_insensitive=False, search_dir=BIOS,
@@ -314,19 +332,23 @@ def amiberry(read: Read, runemu: Runemu) -> list[Launcher]:
else:
test = _shell_test(script, condition_line)
condition = test
launchers.append(Launcher(
target = re.search(r'"?(\$\{?\w+\}?)"?\s*$', line)
if target is None:
raise script.missing(f"destination of the copy at line {number}")
launchers.append(_evidence(read, Launcher(
emulator="AMIBERRY", build="standalone", kind="fixed_name",
slot=copied.group(1).removesuffix("-default.uae").removeprefix("Amiga"),
names=[found[0].value.removeprefix("$(FILE_PATH)/")], case_insensitive=False,
search_dir=BIOS,
passes_to=f"{copied.group(1)} kickstart_rom_file, $(FILE_PATH) = {rom_path[0].value}",
fails_when_missing=False,
depends_on_option=condition,
depends_on_option=f"{condition}, {content}",
source_ref=source_ref([
(RUNEMU, [table_line]), (path, [condition_line, number]),
(config, [rom_path[0].line, found[0].line]),
]),
))
), script.expand(target.group(1), number, set()), ["kickstart_rom_file"],
script, copied.group(1)))
if len(launchers) == 1:
raise script.missing("cp of a default .uae config")
return launchers
@@ -420,9 +442,13 @@ def easyrpg(read: Read) -> list[Launcher]:
def extract_launchers(read: Read) -> list[dict[str, Any]]:
"""Every launcher record, read through `read(repo-relative path)`."""
runemu = Runemu(read)
# The atari800.cfg AmberELEC ships names the OS ROMs, but the core opens
# it only under its atari800_cfg option, off by default, and finds the
# ROMs by a scan of the content folder (libretro-atari800 atari.c:512-516,
# 537, libretro_core_options.h:1203): no record reads it.
launchers = [
*retroarch(read), *retrorun(read), *mame(runemu),
*atari800(read), *scummvm(read), *freej2me(read), *easyrpg(read),
*scummvm(read), *freej2me(read), *easyrpg(read),
*amiberry(read, runemu), *hatari(read, runemu), *pico8(read, runemu),
*piemu(read, runemu), *yabasanshiro(read, runemu),
]
+10 -1
View File
@@ -39,6 +39,14 @@ class Launcher:
machines: list[str] | None = None
# The platform systems the record is written for, none for every one.
systems: list[str] | None = None
# For a record citing a config: where the platform's code puts it
# ({path, source_ref}), whether that leaves a file already there, and
# the launch code rewriting its keys ([] when none does).
installs: dict[str, str] | None = None
kept_if_present: bool | None = None
overrides: list[str] | None = None
# The feature or option file listing the values the option written takes.
choices_from: str | None = None
def entry(self) -> dict[str, Any]:
"""The platform YAML form, in a fixed key order, unset keys left out."""
@@ -46,7 +54,8 @@ class Launcher:
"emulator", "systems", "build", "kind", "slot", "names", "expands",
"case_insensitive", "search_dir", "passes_to",
"fails_when_missing", "fails_scope", "depends_on_option",
"device", "machines", "source_ref",
"device", "machines", "installs", "kept_if_present", "overrides",
"choices_from", "source_ref",
)
return {key: getattr(self, key) for key in order if getattr(self, key) is not None}
+51 -12
View File
@@ -44,6 +44,8 @@ BIOS = "${biosPath}"
EMULATION_BIOS = "${emulationPath}/bios"
STORAGE = "${storagePath}"
FLATPAK_DATA = "${HOME}/.var/app/"
# The function every EmuScript installs its Flatpak config with.
CONFIG_EMU_FP = "functions/configEmuFP.sh"
# EmuScripts file names that differ from the emulator profile they set up.
# The platform and target scrapers both read them here.
@@ -95,6 +97,31 @@ def _is_bios(value: str) -> bool:
return any(value == root or value.startswith(root + "/") for root in (BIOS, EMULATION_BIOS))
def with_evidence(tree: RepoTree, script: Script, record: Launcher, config: str) -> Launcher:
"""A record citing a shipped config, with the steps that install it.
configEmuFP rsyncs the repository's configs/<app> over the Flatpak's:
the _init call overwrites (keeping a .bak), every other call passes
--ignore-existing and leaves the user's file in place.
"""
fp = Script(CONFIG_EMU_FP, tree.read(CONFIG_EMU_FP))
rsync_line, _ = fp.find(r"^\s*rsync .*\$emudeckBackend/configs/", "rsync of the configs")
keep_line, _ = fp.find(r'overwrite="--ignore-existing"', "--ignore-existing")
calls = {n for n, line in script.code() if re.match(r"\s*configEmuFP\s", line)}
if not calls:
raise script.missing("configEmuFP call")
variable = script.assignment_ending("/" + config)
if variable is None:
raise script.missing(f"config variable for {config}")
record.installs = {
"path": script.expand(variable.value, variable.line, set()),
"source_ref": source_ref([(script.path, calls), (fp.path, {keep_line, rsync_line})]),
}
record.kept_if_present = True
record.overrides = []
return record
def _repo_config(script: Script, path: str) -> str:
"""The repository copy of a Flatpak config EmuDeck installs."""
if not path.startswith(FLATPAK_DATA):
@@ -268,7 +295,8 @@ def ares(tree: RepoTree) -> list[Launcher]:
raise LauncherPatternMissing(path, f"Firmware path under {old}")
for system, found in entries.items():
folder = posixpath.commonpath([posixpath.dirname(t) for _, t, _ in found])
records.append(
records.append(with_evidence(
tree, script,
Launcher(
emulator="ares",
build="standalone",
@@ -281,16 +309,20 @@ def ares(tree: RepoTree) -> list[Launcher]:
search_dir=folder,
passes_to="settings.bml "
+ ", ".join(f"{system}/Firmware/{k}" for k, _, _ in found),
)
)
),
posixpath.basename(path),
))
return records
def melonds(tree: RepoTree) -> list[Launcher]:
"""melonDS.ini paths set by changeLine, melonDS.toml paths shipped and rewritten by sed."""
name = "emuDeckMelonDS.sh"
records = setter_records(tree, name, option_key="ExternalBIOSEnable")
script = _script(tree, name)
records = [
with_evidence(tree, script, record, str(record.passes_to or "").split(" ", 1)[0])
for record in setter_records(tree, name, option_key="ExternalBIOSEnable")
]
# A config whose BIOS keys the setters write again keeps their values.
set_by_setters = {file for file, _, _, value, _ in setters(script) if _is_bios(value)}
for file, old, new, sed_lines in sed_rewrites(script, "melonDS_init"):
@@ -306,22 +338,29 @@ def melonds(tree: RepoTree) -> list[Launcher]:
continue
folder = posixpath.commonpath([posixpath.dirname(t) for _, t in found])
config = posixpath.basename(path)
option = next((s for s in settings(tree.read(path)) if s.key == "ExternalBIOSEnable"), None)
if option is None:
raise LauncherPatternMissing(path, "ExternalBIOSEnable")
shipped = {s.key: s for s in settings(tree.read(path))}
if "ExternalBIOSEnable" not in shipped or "ConsoleType" not in shipped:
raise LauncherPatternMissing(path, "ExternalBIOSEnable and ConsoleType")
option, console = shipped["ExternalBIOSEnable"], shipped["ConsoleType"]
# The DS keys are read under ExternalBIOSEnable, the DSi keys only
# when the console is a DSi (ConsoleType 1).
record = Launcher(
emulator="melonds",
build="standalone",
kind="fixed_name",
names=[posixpath.relpath(t, folder) for _, t in found],
source_ref=source_ref(
[(path, {option.line, *(s.line for s, _ in found)}), (script.path, sed_lines)]
),
source_ref=source_ref([
(path, {option.line, console.line, *(s.line for s, _ in found)}),
(script.path, sed_lines),
]),
search_dir=folder,
passes_to=describe_keys(config, tuple((s.section, s.key) for s, _ in found)),
depends_on_option=f"ExternalBIOSEnable = {option.value} in the shipped {config}",
depends_on_option=(
f"[DS] keys: ExternalBIOSEnable = true, {option.value} in the shipped {config}; "
f"[DSi] keys: ConsoleType = 1, {console.value} in the shipped {config}"
),
)
records.append(record)
records.append(with_evidence(tree, script, record, config))
if len(records) < 2:
raise script.missing("melonDS.toml BIOS paths rewritten by sed")
return records
+67 -52
View File
@@ -14,7 +14,7 @@ from __future__ import annotations
import posixpath
import re
from collections.abc import Callable
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from .configgen_launchers import Launcher
@@ -22,6 +22,8 @@ from .rocknix_launchers import fixed_paths
from .shell_launchers import (
LauncherPatternMissing,
Link,
config_placement,
evidence_fields,
RepoTree,
Script,
describe_keys,
@@ -104,6 +106,28 @@ class Runs:
return (RUN_EMU, set(self.of(name).lines))
def _evidence(
record: Launcher,
script: Script,
target: str,
keys: list[str],
source_name: str = "",
rewrites: Iterable[int] = (),
) -> Launcher:
"""A config record with the start script lines that put its file in place.
``rewrites`` are lines the extractor read as rewriting the record's
values without naming their keys (advmame.sh's sed of /roms/mame).
"""
placement = config_placement(script, target, keys, source_name=source_name)
placement.overrides |= set(rewrites)
fields = evidence_fields([placement])
record.installs = fields["installs"]
record.kept_if_present = fields["kept_if_present"]
record.overrides = fields["overrides"]
return record
def _read(tree: RepoTree, path: str) -> Script:
return Script(path, tree.read(path))
@@ -317,7 +341,7 @@ def duckstation(tree: RepoTree, runs: Runs) -> list[Launcher]:
name = posixpath.basename(script.path)
lines = {found[k].line for k in (folder_key, *file_keys) if k in found}
unset = describe_keys("", file_keys).strip()
return [
return [_evidence(
Launcher(
emulator=runs.emulator(name),
build="standalone",
@@ -328,8 +352,11 @@ def duckstation(tree: RepoTree, runs: Runs) -> list[Launcher]:
),
search_dir=found[folder_key].value,
passes_to=f"settings.ini [BIOS] SearchDirectory (-settings), {unset} unset",
)
]
),
script,
script.expand("${CONFIG_DIR}/settings.ini", run_line, set()),
[f"[{s}] {k}" for s, k in (folder_key, *file_keys)],
)]
YABASANSHIRO_SCRIPTS = (
@@ -473,7 +500,7 @@ def hatari(tree: RepoTree, runs: Runs) -> list[Launcher]:
mk = _read(tree, f"{package}/package.mk")
mk_line, _ = mk.find(r"cp -R \$\{PKG_DIR\}/config/\* ", "config install")
name = posixpath.basename(script.path)
return [
return [_evidence(
Launcher(
emulator=runs.emulator(name),
build="standalone",
@@ -485,8 +512,12 @@ def hatari(tree: RepoTree, runs: Runs) -> list[Launcher]:
),
search_dir=value.search_dir,
passes_to=f"{describe_keys(copy.group(1), value.keys)} (--configfile)",
)
]
),
script,
script.expand("${" + copy.group(2) + "}", copy_line, set()),
[f"[{s}] {k}" if s else k for s, k in value.keys],
source_name=copy.group(1),
)]
_ADVMAME_SYSTEM = re.compile(r'^\s*if \[\[ "\$\{1\}" = "([\w-]+)" \]\]; then$')
@@ -535,29 +566,34 @@ def advancemame(tree: RepoTree, runs: Runs) -> list[Launcher]:
(script.path, {run_line, *lines}),
])
found = []
target = script.expand("${CONFIG_DIR}/advmame.rc", run_line, set())
def proved(record: Launcher, key: str) -> Launcher:
return _evidence(record, script, target, [key], rewrites={sed_line})
records = []
for key, folder in value:
rewritten = folder.replace(sed.group(1), sed.group(2))
if rewritten != folder:
# Every system but the one the script rewrites the key for, and
# that system with the rewritten folder.
found.append(Launcher(
records.append(proved(Launcher(
emulator=runs.emulator(name), build="standalone", kind="passes_path", names=[],
source_ref=refs(test_line, sed_line), search_dir=folder,
passes_to=f"advmame.rc {key}",
depends_on_option=f"system != {test.group(1)}",
))
found.append(Launcher(
), key))
records.append(proved(Launcher(
emulator=runs.emulator(name), build="standalone", kind="passes_path", names=[],
source_ref=refs(test_line, sed_line), search_dir=rewritten,
passes_to=f"advmame.rc {key}", systems=[test.group(1)],
))
), key))
continue
found.append(Launcher(
records.append(proved(Launcher(
emulator=runs.emulator(name), build="standalone", kind="passes_path", names=[],
source_ref=refs(), search_dir=folder, passes_to=f"advmame.rc {key}",
))
return found
), key))
return records
def mame(tree: RepoTree, runs: Runs) -> list[Launcher]:
@@ -594,42 +630,11 @@ def mame(tree: RepoTree, runs: Runs) -> list[Launcher]:
)
if not records:
raise script.missing(f"copy into {BIOS_ROOT}")
ini = next((r for r in records if r.names == ["mame*"]), None)
if ini is None:
raise script.missing("copy of mame.ini")
option_path = f"{package}/config/MAME/MAME.opt"
option = next(
(s for s in settings(tree.read(option_path)) if s.key == "mame_read_config"), None
)
if option is None:
raise LauncherPatternMissing(option_path, "mame_read_config")
ini_path = f"{package}/config/ini/mame.ini"
# MAME reads from the folders its *path options name and writes to its
# *_directory ones.
folders = [
(s, [p for p in s.value.split(";") if _under(p)])
for s in settings(tree.read(ini_path), separator=" ")
if s.key.endswith("path")
]
folders = [(s, parts) for s, parts in folders if parts]
if not folders:
raise LauncherPatternMissing(ini_path, f"path under {BIOS_ROOT}")
for setting, parts in folders:
for part in parts:
records.append(
Launcher(
emulator=emulator,
build="libretro",
kind="passes_path",
names=[],
source_ref=source_ref(
[(ini_path, {setting.line}), (option_path, {option.line})]
) + f", {ini.source_ref}",
search_dir=part.rstrip("/"),
passes_to=f"mame.ini {setting.key}",
depends_on_option=f"mame_read_config = {option.value}",
)
)
# mame.ini hands the core no folder: MAME.opt turns on
# mame_mame_paths_enable, so the core passes no path option
# (libretro mame src/osd/libretro/libretro-internal/retro_init.cpp:554-555),
# and the ini MAME would read sits under a working directory no launcher
# sets (src/emu/emuopts.cpp:49). Only the copies are the launcher's.
return records
@@ -644,6 +649,16 @@ def scummvm(tree: RepoTree, runs: Runs) -> list[Launcher]:
core_line, core = script.find(r"-L /tmp/cores/(\w+)_libretro\.so", "libretro start")
arm_line, arm = script.find(r'^\s*"(\w+)"\)', "setup arm", core_line)
copy_line, copy = script.find(r"^\s*cp (\S+) (\S+)$", "copy into the BIOS folder", arm_line)
launched = sorted({
m.group(1) for _, line in runs.script.code()
if (m := re.search(rf"\b{re.escape(name)} (\w+)", line))
})
if not launched:
raise runs.script.missing(f"start of {name} with an action")
action = f"{name} {arm.group(1)}"
if arm.group(1) not in launched:
# The copy runs from the setup action, never from a launch.
action += f", an action no launch runs ({RUN_EMU.rsplit('/', 1)[-1]} passes {', '.join(launched)})"
used = {copy_line}
source = script.expand(copy.group(1), copy_line, used)
target = script.expand(copy.group(2), copy_line, used)
@@ -669,7 +684,7 @@ def scummvm(tree: RepoTree, runs: Runs) -> list[Launcher]:
search_dir=posixpath.dirname(source),
passes_to=target,
fails_when_missing=False,
depends_on_option=f"{name} {arm.group(1)}",
depends_on_option=action,
),
]
+12 -4
View File
@@ -1001,6 +1001,14 @@ def boot_disks(
emulator = (
", ".join(_caller_cores(cg, spec.caller)) if spec.caller else cg.emulator(module)
)
if spec.caller and spec.feature[:2] == ("libretro", "cores"):
# The option is offered for one core of the system: es_features
# lists it under that core, the other cores the generator serves
# never see it (KNULLI's macintosh offers mame alone).
core = spec.feature[2]
if core not in emulator.split(", "):
src.fail(f"es_features offers the option for {core}, which the generator does not serve")
emulator = core
found = 0
for chain in ast.walk(src.tree):
if not (
@@ -1054,10 +1062,10 @@ def boot_disks(
expands={option: values}, case_insensitive=False, search_dir=cg.bios,
passes_to=f"command line {flag}", fails_when_missing=False,
depends_on_option=f"{option} ({' and '.join(tests)})",
source_ref=", ".join([
src.ref([*spans, src.span(statement), _lines(added)]),
f"{ES_FEATURES}:{lines[0]}-{lines[1]}" if not listed else "",
]).rstrip(", "),
# The values the menu offers come from es_features.yml;
# the generator writing the choice is the evidence.
choices_from=None if listed else f"{ES_FEATURES}:{lines[0]}-{lines[1]}",
source_ref=src.ref([*spans, src.span(statement), _lines(added)]),
))
if not found:
src.fail("names no BIOS file from an option value")
+35 -3
View File
@@ -14,6 +14,11 @@ import bisect
import re
from collections.abc import Callable, Iterator
try:
from .shell_launchers import is_feature_ref, split_ref
except ImportError:
from shell_launchers import is_feature_ref, split_ref # type: ignore[no-redef]
GENERATORS = "emulatorLauncher/Generators"
FEATURES = ".emulationstation/es_features.cfg"
@@ -1254,6 +1259,11 @@ def xemu(sources: dict[str, Source]) -> list[Record]:
)
flash = src.find(r'isOptSet\("(xemu_flashrom)"\)', "xemu_flashrom")
# The flashrom is written outside a Chihiro launch only.
chihiro_set = src.find(r"(_chihiro) = \(([^;]+)\);", "Chihiro test")
chihiro_guard = src.find_all(
rf"if \(!{chihiro_set.group(1)}\)", "flashrom guard", end=flash.start()
)[-1]
flash_default = src.find(
r'ini\.WriteValue\("([\w.]+)", "(flashrom_path)", "\'" \+ Path\.Combine\(flashromPath, "([^"]+)"\)',
"flashrom default",
@@ -1267,7 +1277,10 @@ def xemu(sources: dict[str, Source]) -> list[Record]:
"core_option",
_unique([flash_default.group(3), *choices]),
source_ref(
(src.path, [src.lines(flash.start(), flash_default.end())]),
(src.path, [
src.lines(chihiro_set.start(), chihiro_set.end()),
src.lines(chihiro_guard.start(), flash_default.end()),
]),
(features.path, [choice_lines]),
),
case_insensitive=True,
@@ -1277,7 +1290,7 @@ def xemu(sources: dict[str, Source]) -> list[Record]:
f" default {flash_default.group(3)}"
),
fails_when_missing="path written without an existence check",
depends_on_option=flash.group(1),
depends_on_option=f"{flash.group(1)}, not ({chihiro_set.group(2)})",
)
)
@@ -3890,4 +3903,23 @@ def extract_launchers(texts: dict[str, str]) -> list[Record]:
if missing:
raise LauncherPatternMissing(missing[0], "source file")
sources = {path: Source(path, text) for path, text in texts.items()}
return [record for extractor in EXTRACTORS for record in extractor(sources)]
return [
_choices_apart(record) for extractor in EXTRACTORS for record in extractor(sources)
]
def _choices_apart(record: Record) -> Record:
"""The es_features.cfg part of a record's source_ref, moved to choices_from.
The feature file lists the values a menu option offers. The generator
writing the choice is the record's evidence, never the list.
"""
parts = split_ref(str(record["source_ref"]))
listed = [p for p in parts if is_feature_ref(p)]
if not listed:
return record
code = [p for p in parts if not is_feature_ref(p)]
if not code:
raise LauncherPatternMissing(listed[0], "generator code beside the option list")
moved = {k: v for k, v in record.items() if k != "source_ref"}
return {**moved, "choices_from": ", ".join(listed), "source_ref": ", ".join(code)}
+43 -5
View File
@@ -35,6 +35,8 @@ PLACEHOLDER = "RETRODECKBIOSDIR"
# gzdoom/component_prepare.sh:27 rewrites to $roms_path.
ROMS_PLACEHOLDER = "RETRODECKROMSDIR"
CORE_OPTIONS = "retroarch/rd_assets/rd_config/retroarch-core-options.cfg"
# XRoar ROM lists RetroDECK's xroar.conf defines that no machine looks up.
UNREAD_ROMLISTS = frozenset({"coco1", "coco1e", "coco1e_ext", "coco2", "coco2_ext", "coco2b"})
# The component that runs libretro cores.
LIBRETRO_COMPONENT = "retroarch"
# Values of a core option that name a file rather than a setting.
@@ -301,6 +303,19 @@ class Component:
if copied is None:
raise LauncherPatternMissing(f"{script.path}:{number}", f"no preset copied to {target}")
preset = f"{self.name}/rd_assets/rd_config/{copied[1]}"
# The reset copies the preset over the user's file, then fills its
# placeholders: that is the install, and nothing keeps an older file.
evidence: dict[str, Any] = {
"installs": {
"path": target,
"source_ref": source_ref(
[(script.path, copied[0])]
+ [(script.path, line) for _v, line in substitutions.values()]
),
},
"kept_if_present": False,
"overrides": self.launch_rewrites(target),
}
text = self.read(preset)
if text is None:
raise LauncherPatternMissing(preset, "preset is missing")
@@ -332,10 +347,10 @@ class Component:
self.record(
"passes_path", [], [(preset, line_no)] + applied,
search_dir=value, passes_to=f"{self.config_name(target)} {where}",
fails_when_missing=False,
fails_when_missing=False, **evidence,
)
if key == "rompath":
self.romlists(preset, text, value, applied)
self.romlists(preset, text, value, applied, evidence)
if not found:
raise LauncherPatternMissing(preset, f"no {PLACEHOLDER} line")
@@ -349,10 +364,28 @@ class Component:
found[m.group(1)] = (f"${m.group(2)}", line_no)
return found
def launch_rewrites(self, target: str) -> list[str]:
"""The sed lines of the component launcher rewriting ``target``."""
path = f"{self.name}/component_launcher.sh"
text = self.read(path)
if text is None:
return []
return [
source_ref([(path, number)])
for number, line in enumerate(text.splitlines(), 1)
if re.search(r"\bsed\b.*\s-i\b", line) and target in line
]
def romlists(
self, preset: str, text: str, rompath: str, applied: list[tuple[str, int]]
self, preset: str, text: str, rompath: str, applied: list[tuple[str, int]],
evidence: dict[str, Any],
) -> None:
"""XRoar's `romlist NAME=a,b,@other`: the names tried for each ROM, in order."""
"""XRoar's `romlist NAME=a,b,@other`: the names tried for each ROM, in order.
A list no XRoar machine or cartridge looks up is no read: coco1,
coco1e, coco1e_ext, coco2, coco2_ext and coco2b are defined by XRoar
(src/xroar.c:663-670) and named by nothing else.
"""
lists: dict[str, tuple[int, list[str]]] = {}
for line_no, line in enumerate(text.splitlines(), 1):
found = re.match(r"^romlist\s+([\w-]+)=(.*)$", line.strip())
@@ -375,10 +408,12 @@ class Component:
return list(dict.fromkeys(names))
for name, (line_no, _items) in lists.items():
if name in UNREAD_ROMLISTS:
continue
self.record(
"names_searched", expand(name, (name,)), [(preset, line_no)] + applied,
slot=name, search_dir=rompath, passes_to=f"romlist {name}",
fails_when_missing=False,
fails_when_missing=False, **evidence,
)
def read_launcher(self) -> None:
@@ -480,6 +515,7 @@ def core_options(read: Read) -> list[Launcher]:
]
if not copied:
raise LauncherPatternMissing(prepare_path, "the core options preset is never copied")
target = _words(prepare.splitlines()[copied[0] - 1].strip())[-1]
records = []
for line_no, line in enumerate(text.splitlines(), 1):
found = re.match(r'^([\w.-]+)\s*=\s*"([^"]*)"$', line.strip())
@@ -489,6 +525,8 @@ def core_options(read: Read) -> list[Launcher]:
records.append(Launcher(
emulator=LIBRETRO_COMPONENT, build="libretro", kind="core_option", names=[value],
passes_to=f"core option {key}", fails_when_missing=False,
installs={"path": target, "source_ref": source_ref([(prepare_path, copied[0])])},
kept_if_present=False, overrides=[],
source_ref=source_ref([(CORE_OPTIONS, line_no), (prepare_path, copied[0])]),
))
if not records:
+7 -1
View File
@@ -94,6 +94,11 @@ class Launcher:
passes_to: str | None = None
fails_when_missing: bool | str | None = None
depends_on_option: str | None = None
# For a record citing a config: where the platform's code puts it, whether
# that leaves a file already there, and the launch code rewriting it.
installs: dict[str, str] | None = None
kept_if_present: bool | None = None
overrides: list[str] | None = None
lines: list[tuple[str, int]] = field(default_factory=list, repr=False)
def entry(self) -> dict[str, Any]:
@@ -101,7 +106,8 @@ class Launcher:
order = (
"emulator", "package", "repo", "systems", "build", "kind", "slot",
"names", "case_insensitive", "search_dir", "passes_to",
"fails_when_missing", "depends_on_option", "source_ref",
"fails_when_missing", "depends_on_option", "installs", "kept_if_present",
"overrides", "source_ref",
)
return {key: getattr(self, key) for key in order if getattr(self, key) is not None}
+149 -28
View File
@@ -18,6 +18,7 @@ from dataclasses import dataclass
try:
from .configgen_launchers import Launcher
from .shell_launchers import config_placement, evidence_fields
from .shell_launchers import (
LauncherPatternMissing,
Link,
@@ -30,6 +31,7 @@ try:
)
except ImportError:
from configgen_launchers import Launcher
from shell_launchers import config_placement, evidence_fields # type: ignore[no-redef]
from shell_launchers import (
LauncherPatternMissing,
Link,
@@ -46,6 +48,9 @@ DEVICES = f"{ROCKNIX}/devices"
STANDALONE = f"{ROCKNIX}/packages/emulators/standalone"
RETROARCH = f"{ROCKNIX}/packages/emulators/libretro/retroarch"
BIOS_ROOT = "/storage/roms/bios"
CONFIG_ROOT = "/storage/.config"
# The first-boot step copying every package's /usr/config to /storage/.config.
USERCONFIG = f"{ROCKNIX}/packages/sysutils/systemd/scripts/userconfig-setup"
Lines = list[tuple[str, set[int]]]
@@ -186,6 +191,38 @@ def bios_keys(tree: RepoTree, paths: list[str], root: str = BIOS_ROOT) -> list[t
return keys
def with_evidence(
tree: RepoTree,
record: Launcher,
target: str,
keys: list[str],
start: Script | None = None,
writers: tuple[str, ...] = (),
source_name: str = "",
) -> Launcher:
"""A config record with the steps that put its file at ``target``.
The first boot copies /usr/config to /storage/.config, leaving a file
already there. A start script can copy or generate it again, and its
sed lines rewriting one of the record's keys are the overrides.
"""
setup = Script(USERCONFIG, tree.read(USERCONFIG))
placements = [config_placement(setup, target, required=False, sources=("/usr/config/",))]
if start is not None:
placements.append(config_placement(
start, target, keys, writers, source_name, required=False
))
fields = evidence_fields(placements)
record.installs = fields["installs"]
record.kept_if_present = fields["kept_if_present"]
record.overrides = fields["overrides"]
return record
def _keys(value: FixedPaths) -> list[str]:
return [f"[{section}] {key}" if section else key for section, key in value.keys]
def config_records(
tree: RepoTree,
emulator: str,
@@ -193,10 +230,17 @@ def config_records(
file: str,
passes_to_suffix: str = "",
depends_on_option: str | None = None,
target: str = "",
start: Script | None = None,
unread: Callable[[tuple[str, str]], bool] = lambda key: False,
) -> list[Launcher]:
"""fixed_name records for the bios paths of a config installed per device."""
"""fixed_name records for the bios paths of a config installed per device.
``unread`` drops the keys the shipped configs leave the emulator
reading no file from.
"""
configs = device_configs(tree, package, file)
keys = bios_keys(tree, [c.path for c in configs])
keys = [k for k in bios_keys(tree, [c.path for c in configs]) if not unread(k)]
name = posixpath.basename(file)
def read(path: str, text: str) -> tuple[object, set[int]]:
@@ -206,7 +250,8 @@ def config_records(
for device, value, refs in per_device(tree, configs, f"{package}/package.mk", read):
assert isinstance(value, FixedPaths)
described = describe_keys(name, value.keys)
records.append(
records.append(with_evidence(
tree,
Launcher(
emulator=emulator,
build="standalone",
@@ -219,8 +264,11 @@ def config_records(
),
device=device,
depends_on_option=depends_on_option,
)
)
),
target or f"{CONFIG_ROOT}/{posixpath.basename(package)}/{name}",
_keys(value),
start,
))
return records
@@ -241,7 +289,8 @@ def retroarch(tree: RepoTree) -> list[Launcher]:
for device, value, refs in per_device(tree, configs, f"{RETROARCH}/package.mk", read):
assert isinstance(value, tuple)
values = dict(value)
records.append(
records.append(with_evidence(
tree,
Launcher(
emulator="retroarch",
build="libretro",
@@ -251,8 +300,10 @@ def retroarch(tree: RepoTree) -> list[Launcher]:
search_dir=values["system_directory"],
passes_to="retroarch.cfg " + ", ".join(f'{k} = "{v}"' for k, v in value),
device=device,
)
)
),
f"{CONFIG_ROOT}/retroarch/retroarch.cfg",
[k for k, _ in value],
))
return records
@@ -275,6 +326,19 @@ def amiberry(tree: RepoTree) -> list[Launcher]:
)
used: set[int] = {test_line, then_line, else_line, other_line, sed_line}
root = script.expand("${" + sed.group(2) + "}", sed_line, used)
written = re.search(r'"?(\$\{?\w+\}?)"?\s*$', script.lines[then_line - 1])
if written is None:
raise script.missing("destination of the config copy")
target = script.expand(written.group(1), then_line, set())
# A WHDLoad archive is booted by --autoload, which builds its own setup
# (kickstart included) over the config: the record holds for floppies.
autoload = [n for n, line in script.code() if re.search(r"amiberry .*--autoload", line)]
if not autoload:
raise script.missing("WHDLoad boot with --autoload")
content = (
"content not booted with --autoload, a WHDLoad .lha or .zip "
f"({posixpath.basename(script.path)}:{', '.join(map(str, autoload))})"
)
records = []
for config, condition in (
(then.group(1), f"game name matches {test.group(1)}"),
@@ -290,7 +354,8 @@ def amiberry(tree: RepoTree) -> list[Launcher]:
raise LauncherPatternMissing(path, f"key naming a file under {sed.group(1)}")
value, lines = fixed_paths(found, keys, path)
slot = re.fullmatch(r"Amiga(\w+)-default\.uae", config)
records.append(
records.append(with_evidence(
tree,
Launcher(
emulator="amiberry",
build="standalone",
@@ -300,9 +365,13 @@ def amiberry(tree: RepoTree) -> list[Launcher]:
slot=slot.group(1) if slot else config,
search_dir=value.search_dir,
passes_to=f"{describe_keys(config, value.keys)} (amiberry -f)",
depends_on_option=condition,
)
)
depends_on_option=f"{condition}, {content}",
),
target,
_keys(value),
script,
source_name=config,
))
mk = Script(f"{package}/package.mk", tree.read(f"{package}/package.mk"))
links = mk.links(BIOS_ROOT, fixed={"INSTALL": ""})
if not links:
@@ -375,7 +444,11 @@ def hatari(tree: RepoTree) -> list[Launcher]:
if not keys:
raise LauncherPatternMissing(path, f"key naming a file under {BIOS_ROOT}")
value, lines = fixed_paths(found, keys, path)
return [
copied = re.search(r'"?(\$\{?\w+\}?)"?\s*$', script.lines[copy_line - 1])
if copied is None:
raise script.missing("destination of the default config copy")
return [with_evidence(
tree,
Launcher(
emulator="hatari",
build="standalone",
@@ -384,8 +457,12 @@ def hatari(tree: RepoTree) -> list[Launcher]:
source_ref=source_ref([(path, lines), (script.path, {copy_line, run_line})]),
search_dir=value.search_dir,
passes_to=f"{describe_keys(copy.group(1), value.keys)} (--configfile)",
)
]
),
script.expand(copied.group(1), copy_line, set()),
_keys(value),
script,
source_name=copy.group(1),
)]
def melonds(tree: RepoTree) -> list[Launcher]:
@@ -406,7 +483,22 @@ def melonds(tree: RepoTree) -> list[Launcher]:
rf"^\s*{test.group(1)}=\$\(get_setting (\w+) ", f"setting read into {test.group(1)}"
)
condition = f"{setting.group(1)} = {test.group(2)} sets ExternalBIOSEnable=1, otherwise 0"
records = config_records(tree, "melonds", package, "melonDS.ini", depends_on_option=condition)
configs = [c.path for c in device_configs(tree, package, "melonDS.ini")]
# melonDS opens the DSi BIOS and firmware keys only under
# DSiExternalBIOSEnable, off unless a config sets it.
dsi_external = any(
s.key == "DSiExternalBIOSEnable" and s.value not in ("", "0")
for path in configs for s in settings(tree.read(path))
)
def unread(key: tuple[str, str]) -> bool:
return not dsi_external and key[1] in ("DSiBIOS9Path", "DSiBIOS7Path", "DSiFirmwarePath")
conf_line, conf = script.find(r'^\s*CONF_DIR="([^"]+)"', "CONF_DIR")
records = config_records(
tree, "melonds", package, "melonDS.ini", depends_on_option=condition,
target=f"{conf.group(1)}/melonDS.ini", start=script, unread=unread,
)
for record in records:
record.source_ref += (
f", {source_ref([(script.path, {setting_line, test_line, enable_line})])}"
@@ -420,7 +512,10 @@ def dsperate(tree: RepoTree) -> list[Launcher]:
f"{package}/scripts/start_dsperate.sh", tree.read(f"{package}/scripts/start_dsperate.sh")
)
run_line, _ = script.find(r"/usr/bin/dsperate --config ", "dsperate --config")
records = config_records(tree, "dsperate", package, "dsperate.ini", " (--config)")
records = config_records(
tree, "dsperate", package, "dsperate.ini", " (--config)",
target=script.expand("${DSPERATE_INI}", run_line, set()), start=script,
)
for record in records:
record.source_ref += f", {source_ref([(script.path, {run_line})])}"
return records
@@ -434,7 +529,10 @@ def xemu(tree: RepoTree) -> list[Launcher]:
copy_line, _ = script.find(
r'cp -r "\$\{SOURCE_DIR\}/\$\{XEMU_INI\}"', "xemu.toml copy at launch"
)
records = config_records(tree, "xemu", package, "xemu.toml", " (copied at each launch)")
records = config_records(
tree, "xemu", package, "xemu.toml", " (copied at each launch)",
target=script.expand("${CONF_DIR}/${XEMU_INI}", copy_line, set()), start=script,
)
for record in records:
record.source_ref += f", {source_ref([(script.path, {copy_line})])}"
return records
@@ -452,7 +550,9 @@ def nanoboyadvance(tree: RepoTree) -> list[Launcher]:
mk_line, _ = Script(mk_path, tree.read(mk_path)).find(
r"config/config\.toml", "config.toml install"
)
return [
start_path = f"{package}/scripts/start_nanoboyadvance.sh"
return [with_evidence(
tree,
Launcher(
emulator="nanoboyadvance",
build="standalone",
@@ -461,8 +561,14 @@ def nanoboyadvance(tree: RepoTree) -> list[Launcher]:
source_ref=source_ref([(path, lines), (mk_path, {mk_line})]),
search_dir=value.search_dir,
passes_to=describe_keys("config.toml", value.keys),
)
]
# NanoBoyAdvance exits when its BIOS cannot be read
# (src/platform/sdl/main.cpp:180-185 at the build ROCKNIX ships).
fails_when_missing=True,
),
f"{CONFIG_ROOT}/nanoboyadvance/config.toml",
_keys(value),
Script(start_path, tree.read(start_path)),
)]
def mednafen(tree: RepoTree) -> list[Launcher]:
@@ -486,7 +592,9 @@ def mednafen(tree: RepoTree) -> list[Launcher]:
modules: dict[str, list[Setting]] = {}
for setting in firmware:
modules.setdefault(setting.key.split(".", 1)[0], []).append(setting)
return [
home = script.expand("${MEDNAFEN_HOME}", gen_line, set())
return [with_evidence(
tree,
Launcher(
emulator="mednafen",
build="standalone",
@@ -501,9 +609,12 @@ def mednafen(tree: RepoTree) -> list[Launcher]:
slot=module,
search_dir=folder.value,
passes_to="mednafen.cfg " + ", ".join(s.key for s in group),
)
for module, group in modules.items()
]
),
f"{home}/mednafen.cfg",
[s.key for s in group],
script,
writers=("mednafen_gen_config.sh",),
) for module, group in modules.items()]
def yabasanshiro(tree: RepoTree) -> list[Launcher]:
@@ -549,6 +660,8 @@ def directory_records(
file: str,
folder_key: tuple[str, str],
file_keys: tuple[tuple[str, str], ...],
target: str = "",
start: str = "",
) -> list[Launcher]:
"""scans_dir records: the config hands a directory and leaves the file keys empty."""
configs = device_configs(tree, package, file)
@@ -568,7 +681,8 @@ def directory_records(
records = []
for device, folder, refs in per_device(tree, configs, f"{package}/package.mk", read):
passes = f"{name} [{folder_key[0]}] {folder_key[1]} = {folder}, {unset} unset"
records.append(
records.append(with_evidence(
tree,
Launcher(
emulator=emulator,
build="standalone",
@@ -578,8 +692,11 @@ def directory_records(
search_dir=folder,
passes_to=passes,
device=device,
)
)
),
target,
[f"[{s}] {k}" for s, k in (folder_key, *file_keys)],
Script(start, tree.read(start)) if start else None,
))
return records
@@ -591,6 +708,8 @@ def duckstation(tree: RepoTree) -> list[Launcher]:
"settings.ini",
("BIOS", "SearchDirectory"),
(("BIOS", "PathNTSCU"), ("BIOS", "PathNTSCJ"), ("BIOS", "PathPAL")),
f"{CONFIG_ROOT}/duckstation/settings.ini",
f"{STANDALONE}/duckstation-sa/scripts/start_duckstation.sh",
)
@@ -602,6 +721,8 @@ def armsx2(tree: RepoTree) -> list[Launcher]:
"ARMSX2/inis/PCSX2.ini",
("Folders", "Bios"),
(("Filenames", "BIOS"),),
f"{CONFIG_ROOT}/ARMSX2/inis/PCSX2.ini",
f"{STANDALONE}/armsx2-sa/scripts/start_armsx2.sh",
)
+185
View File
@@ -63,6 +63,48 @@ def source_ref(parts: Iterable[tuple[str, Iterable[int]]]) -> str:
)
# The files a platform ships as configuration: an emulator reads them, the
# platform's code never runs them. Batocera's .yml lists and an EmuELEC .opt
# only offer the values a menu option takes.
_CONFIG_FILE = re.compile(
r"\.(?:ini|cfg|conf|rc|rc_\w+|opt|bml|toml|uae|template|yml|yaml)$", re.IGNORECASE
)
_FEATURE_FILE = re.compile(r"(?:^|/)es_features[^/]*$|\.opt$", re.IGNORECASE)
def split_ref(ref: str) -> list[str]:
"""The parts of a source_ref, each "path:lines", a bare line span
continuing the path before it."""
parts: list[str] = []
for part in (p.strip() for p in ref.split(", ")):
if not part:
continue
if re.fullmatch(r"\d+(?:-\d+)?", part) and parts:
parts[-1] += f", {part}"
else:
parts.append(part)
return parts
def ref_path(part: str) -> str:
return part.rsplit(":", 1)[0] if re.search(r":\d", part) else part
def config_ref(ref: str) -> list[str]:
"""The parts of a source_ref that cite a config, option or feature file.
Such a part proves what the file says, never that an emulator reads it:
melonDS.ini keys are written on ROCKNIX for a build that reads them only
when another key is set.
"""
return [part for part in split_ref(ref) if _CONFIG_FILE.search(ref_path(part))]
def is_feature_ref(part: str) -> bool:
"""Whether a source_ref part cites a file that lists a menu option's values."""
return bool(_FEATURE_FILE.search(ref_path(part)))
@dataclass
class Word:
"""A shell word: as written, and with quotes removed."""
@@ -394,6 +436,149 @@ def _merge_links(links: list[Link]) -> list[Link]:
return [link for link in merged if id(link) not in folded]
_COPY = re.compile(r"(?:^|[\s;&|(])(cp|rsync|install|mv)\s")
_GUARD = re.compile(r"\[\[?\s*!\s+-[fde]\s+(\S+)\s*\]")
@dataclass
class ConfigPlacement:
"""How a start script puts a config where the emulator reads it.
``install`` are the lines copying it there, ``kept_if_present`` whether
every such copy runs only when the file (or its folder) is absent, and
``overrides`` the sed lines rewriting one of the record's keys in it.
"""
script: str
path: str
install: set[int]
kept_if_present: bool
overrides: set[int]
def _bare_key(key: str) -> str:
return key.rsplit("] ", 1)[-1].rsplit("/", 1)[-1].strip()
def _guarded(script: Script, number: int, target: str) -> bool:
"""Whether a line runs only when target, or a folder above it, is absent."""
def covers(found: re.Match[str], at: int) -> bool:
guarded = script.expand(words(found.group(1))[0].text, at, set()).rstrip("/")
return target == guarded or target.startswith(guarded + "/")
line = script.lines[number - 1]
if any(covers(m, number) for m in _GUARD.finditer(line)):
return True
depth = 0
for at in range(number - 1, 0, -1):
text = script.lines[at - 1].strip()
if re.match(r"fi\b", text):
depth += 1
elif re.match(r"if\b", text):
if depth:
depth -= 1
continue
return any(covers(m, at) for m in _GUARD.finditer(text))
return False
def config_placement(
script: Script,
target: str,
keys: Iterable[str] = (),
writers: Iterable[str] = (),
source_name: str = "",
required: bool = True,
sources: tuple[str, ...] = (),
) -> ConfigPlacement:
"""The lines of a start script that install ``target`` and rewrite its keys.
A copy installs the target when its destination is the file or a folder
above it. ``writers`` are other commands that write the file (ROCKNIX
generates mednafen.cfg with mednafen_gen_config.sh). A copy guarded by
`[ ! -f X ]` or `[ ! -d X ]` on the target or a folder above it, or made
with cp -n or rsync --ignore-existing, leaves a file already there.
"""
install: set[int] = set()
kept = True
overrides: set[int] = set()
bare = {_bare_key(k) for k in keys} - {""}
writer = re.compile("|".join(re.escape(w) for w in writers)) if writers else None
for number, line in script.code():
if (copy := _COPY.search(line)) or (writer and writer.search(line)):
if copy:
args = []
for word in words(line[copy.start(1):])[1:]:
# The command ends at a separator or a redirection.
if word.raw in {";", "&&", "||", "|"} or re.match(r"\d*>", word.raw):
break
args.append(word.text)
operands = [a for a in args if not a.startswith("-")]
if not operands:
continue
destination = script.expand(operands[-1], number, set()).rstrip("/")
source = script.expand(operands[0], number, set()).rstrip("/")
hits = target == destination or target.startswith(destination + "/") or (
target.startswith(f"{destination}/{source.rsplit('/', 1)[-1]}")
)
# A copy of another file into the same place is another
# record's: amiberry copies the A500 or the A1200 setup.
named = source.rsplit("/", 1)[-1]
if not hits or (source_name and "." in named and named != source_name):
continue
# A named file copied into a folder above the target is
# another file of that folder (ARMSX2's secrets.ini).
if (
not source_name and "." in named and "*" not in named
and destination != target and named != target.rsplit("/", 1)[-1]
):
continue
if sources and not source.startswith(sources):
continue
skip = "-n" in args or "--ignore-existing" in args
else:
skip = False
install.add(number)
if not (skip or _guarded(script, number, target)):
kept = False
elif re.search(r"\bsed\b.*\s-i\b|\bsed -i", line) and bare:
expanded = script.expand(line, number, set())
if target in expanded or target.rsplit("/", 1)[-1] in expanded:
if any(re.search(rf"\b{re.escape(k)}\b", expanded) for k in bare):
overrides.add(number)
if not install and required:
raise script.missing(f"install of {target}")
return ConfigPlacement(script.path, target, install, kept, overrides)
def evidence_fields(
placements: Iterable[ConfigPlacement],
installed: Iterable[tuple[str, Iterable[int]]] = (),
) -> dict:
"""The installs, kept_if_present and overrides fields of a config record.
``placements`` are the steps that put the file in place, in the order
they run (a first-boot copy, then the start script). ``installed`` are
the build steps that put it where they copy it from (a package.mk copy
into /usr/config).
"""
placements = list(placements)
steps = [p for p in placements if p.install]
if not steps:
raise LauncherPatternMissing(placements[0].path if placements else "", "install")
return {
"installs": {
"path": steps[-1].path,
"source_ref": source_ref([*installed, *((p.script, p.install) for p in steps)]),
},
"kept_if_present": all(p.kept_if_present for p in steps),
"overrides": [
source_ref([(p.script, {line})]) for p in placements for line in sorted(p.overrides)
],
}
@dataclass
class Setting:
section: str
+56
View File
@@ -81,7 +81,10 @@ def _validate_yaml_directory(
continue
out.extend(_errors(validator, data, str(path.relative_to(ROOT))))
out.extend(_unreachable_citations(path, data))
if schema_name == "platform.schema.json":
out.extend(_config_records_without_evidence(path, data))
if schema_name == "emulator.schema.json":
out.extend(_config_reads_without_build(path, data))
out.extend(_checks_without_values(path, data))
out.extend(_save_dir_without_library_name(path, data))
out.extend(_systems_outside_profile(path, data))
@@ -102,6 +105,59 @@ _CHECK_FIELDS = {
}
def _config_records_without_evidence(path: Path, document: object) -> list[str]:
"""A launcher record citing a config without the links its extractor reads.
A config file is proof only when the code installing it and the code
reading it are shown (launchers.config_evidence). The platform's half
is the extractor's to write: a record citing a shipped config carries
installs and overrides, and a feature or option file is never a
record's source, only its choices_from.
"""
if not isinstance(document, dict):
return []
from scraper.shell_launchers import config_ref, is_feature_ref # noqa: PLC0415
out: list[str] = []
for index, record in enumerate(document.get("launchers") or []):
if not isinstance(record, dict):
continue
cited = config_ref(str(record.get("source_ref") or ""))
where = f"{path.relative_to(ROOT)}: launchers[{index}] ({record.get('emulator')})"
features = [part for part in cited if is_feature_ref(part)]
if features:
out.append(f"{where} cites {features[0]} as source, choices_from holds it")
if [part for part in cited if not is_feature_ref(part)] and not (
record.get("installs") and "overrides" in record
):
out.append(f"{where} cites a config without installs and overrides")
return out
def _config_reads_without_build(path: Path, document: object) -> list[str]:
"""A config_reads entry naming a platform the profile gives no pin_build.
Such an entry is read at the build that platform ships, and pin_build is
where that build is named.
"""
if not isinstance(document, dict):
return []
builds = {str(b.get("platform")) for b in document.get("pin_build") or []}
out: list[str] = []
for index, read in enumerate(document.get("config_reads") or []):
if not isinstance(read, dict):
continue
if read.get("revision"):
continue
for platform in read.get("platforms") or []:
if str(platform) not in builds:
out.append(
f"{path.relative_to(ROOT)}: config_reads[{index}] ({read.get('file')}) "
f"names {platform}, which has no pin_build"
)
return out
def _systems_outside_profile(path: Path, document: object) -> list[str]:
"""An entry's system: or systems: naming a system its profile does not list.
+19 -1
View File
@@ -100,7 +100,7 @@ from common import (
yaml = require_yaml()
from exporter.candidates import BIOS, EntryCategories
from launchers import LauncherLayer, declared_destinations, held_record
from launchers import LauncherLayer, declared_destinations, held_record, launcher_unproven
from cross_reference import launcher_reads_not_held
from nativemode import (
SIZE_AND_ANY_HASH,
@@ -1764,6 +1764,8 @@ def verify_platform(
"launcher_reads_not_held": launcher_reads_not_held(
{config.get("_key", platform): config}, db, profiles, data_dir_registry
),
# Records a config file does not prove: listed, never consumed.
"launcher_unproven": launcher_unproven({config.get("_key", platform): config}, profiles),
"exclusion_notes": exclusions,
"region_fallbacks": sorted(region_fallbacks),
**({"left_out": left_out} if no_game_data else {}),
@@ -2130,6 +2132,21 @@ def _print_undeclared_section(result: dict, verbose: bool) -> None:
)
def _print_launcher_unproven(result: dict, verbose: bool) -> None:
"""Launcher records resting on a config file the code does not prove."""
rows = result.get("launcher_unproven") or []
if not rows:
return
listed = "" if verbose else " (--verbose lists them)"
print(f" Launcher records unproven: {len(rows)}{listed}")
if not verbose:
return
for row in rows:
print(f" [{row['index']}] {row['emulator']} {row['kind']}: {row['passes_to']}")
for link in row["missing"]:
print(f" missing {link}")
def _print_launcher_reads(result: dict, verbose: bool) -> None:
"""Names the platform's launch code reads that the collection lacks."""
every = result.get("launcher_reads_not_held") or []
@@ -2220,6 +2237,7 @@ def print_platform_result(
_print_detail_entries(result["details"], seen_details, verbose)
_print_undeclared_section(result, verbose)
_print_launcher_reads(result, verbose)
_print_launcher_unproven(result, verbose)
# The same notes come back for every platform running these cores and
# filled 985 of the 1114 lines of verify --all: counted, listed on ask.