kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-01 07:33:45 -06:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge the boot oracle

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-10-27 15:45:52 +01:00
2 parents 7c221c95f8 44ab3febf4
commit 49b47355b9
8 files changed
+1154 -374
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/campaign.py
+323 -178
View File
@@ -1,41 +1,51 @@
#!/usr/bin/env python3
"""Run the real-condition campaign unattended: cores, content, checks, rules.
"""Run the real-condition campaign end to end: cores, content, checks, rules.
For each emulator profile in scope, the campaign takes the core (the libretro
x86_64 Linux nightly, else a build of the profile's pinned commit), the test
content of each of its systems (tests/campaign/content.yml, fetched through
romfetch when it is not on disk), and runs core_probe.py:
A maintainer starts each pass. For each emulator profile in scope, the
campaign takes the core (the libretro x86_64 Linux nightly, else a build of
the profile's pinned commit), the test content of each of its systems
(tests/campaign/content.yml, fetched through romfetch when it is not on
disk), and runs core_probe.py. The boot oracle of core_probe judges every
run: boots, bios_error, crash, hang or unknown, against the reference frames
and error patterns of tests/campaign/oracle.yml.
files every declared file the collection holds, staged, and which ones
the core opens
boot the content with no file against the content with the declared
file, compared on the frames the core shows
files every declared file the collection holds, staged: the full set,
and which files the core opens
none no BIOS at all
negative per declared file of the content's region: the file alone, then
zeros of its size, its first half and other regions' dumps
staged under its name
path a declared file staged where the core looked for it
alias a declared file staged under a name the core looked for
variants each .variants/ copy of a file the core opened
bench the content timed, warm-up excluded, 5 repetitions
bench with --bench only: the content timed, warm-up excluded
Every run is a record in measurements/<emulator>.jsonl. The fixed rule table
(campaign_rules.py) turns records into verdicts. A verdict the table settles
is written into the profile with the record ids as `evidence:`. Any other
goes to measurements/findings.jsonl, where the site lists it as open. Each
pass adds one line to measurements/campaign-runs.jsonl.
(campaign_rules.py) reads the verdicts, never a comparison of frames. A
verdict the table settles is written into the profile with the record ids
as `evidence:`. Any other goes to measurements/findings.jsonl, where the
site lists it as open. Each pass adds one line to
measurements/campaign-runs.jsonl.
`references` runs the full set once per core and content with its frames
saved as PNG, and lists the screens that stay on for several samples: the
candidates a reviewer looks at before writing them into oracle.yml.
State is .cache/campaign/state.json of the main checkout: per check done,
failed or pending with the reason, and the core sha256 and profile digest it
ran against. A pass skips what is done unless the core build or the profile
changed. The campaign holds the host-wide automation lock, waits for an idle
host, stops cleanly on a busy host or a full disk, and resumes on the next
pass. Under autorun (`campaign` job) the lock is the job's.
failed or pending with the reason, and the core sha256, profile digest,
oracle sha256 and record version it ran against. A pass skips what is done
unless one of them changed. The campaign holds the host-wide automation
lock, waits for an idle host, stops cleanly on a busy host or a full disk,
and the next pass a maintainer starts resumes there.
Usage:
python scripts/campaign.py run [--only CORE,...] [--system ID] [--budget 6h]
python scripts/campaign.py run [--only CORE,...] [--system ID] [--budget 6h] [--bench]
python scripts/campaign.py run --dry-run
python scripts/campaign.py run --commit [--commit-every N]
python scripts/campaign.py references --only CORE,... [--system ID]
python scripts/campaign.py status [--json]
python scripts/campaign.py findings [--all] [--json]
python scripts/campaign.py gate
python scripts/campaign.py install-timer [--enable]
"""
from __future__ import annotations
@@ -86,6 +96,7 @@ PENDING = "pending"
CRASH_ATTEMPTS = 3
NETWORK_DELAYS = (10, 60, 300)
MAX_ALIAS_CANDIDATES = 6
MAX_REGION_DUMPS = 2
BENCH_WARMUP = 600
BENCH_FRAMES = 1800
BENCH_REPEAT = 5
@@ -175,6 +186,11 @@ def region_matches(entry: dict, region: str | None) -> bool:
return region in regions or "world" in regions
def regions_of(entry: dict) -> list[str]:
regions = entry.get("region") or []
return [regions] if isinstance(regions, str) else list(regions)
def is_bios(entry: dict) -> bool:
return entry.get("category", "bios") == "bios" and entry.get("type") != "directory"
@@ -235,6 +251,8 @@ class Settings:
bench_frames: int = BENCH_FRAMES
romfetch: str = ""
network: bool = True
# Benchmarks only choose between variants: never part of a default pass.
bench: bool = False
delays: tuple[float, ...] = NETWORK_DELAYS
echo: bool = True
# Third-party code (a nightly core, a project's Makefile) runs only under
@@ -252,6 +270,7 @@ class Paths:
content: Path
db: Path
content_file: Path = CONTENT_FILE
oracle: Path = core_probe.DEFAULT_ORACLE
@property
def state(self) -> Path:
@@ -470,14 +489,17 @@ def save_state(path: Path, state: dict) -> None:
autorun.save_state(path, state)
def should_run(previous: dict | None, core_sha: str, digest: str) -> bool:
def should_run(previous: dict | None, core_sha: str, digest: str, oracle_sha: str = "") -> bool:
"""A check runs again when it never ran, is pending, or ran against
another core build or another version of the profile."""
another core build, another version of the profile, another oracle, or
before records carried a verdict."""
if not previous:
return True
if previous.get("status") == PENDING:
return True
return previous.get("core_sha256") != core_sha or previous.get("profile_digest") != digest
return (previous.get("core_sha256") != core_sha or previous.get("profile_digest") != digest
or previous.get("record_version") != mr.RECORD_VERSION
or previous.get("oracle_sha256", "") != oracle_sha)
# --------------------------------------------------------------------------
@@ -875,6 +897,7 @@ class Campaign:
self.cores = Cores(self)
self.store = ContentStore(self)
self.content = load_content(paths.content_file)
self.oracle_sha = file_sha256(paths.oracle) if paths.oracle.is_file() else ""
self.deadline = 0.0
self.current: Pass | None = None
self.dirty_at_start: set[str] = set()
@@ -1004,6 +1027,8 @@ class Campaign:
"--emulator", emulator, "--subject", subject,
"--max-load", str(self.host.max_load), "--timeout", str(self.settings.probe_timeout),
"--work-dir", str(self.paths.root / "tmp" / "core_probe")]
if self.paths.oracle.is_file():
argv += ["--oracle", str(self.paths.oracle)]
if not self.settings.allow_unsandboxed:
argv.append("--require-sandbox")
if core.artifact:
@@ -1026,7 +1051,8 @@ class Campaign:
def remember(self, emulator: str, key: str, result: ProbeResult, core: Core | None,
digest: str) -> None:
entry = {"status": result.status, "at": now_iso(), "profile_digest": digest,
"core_sha256": core.sha256 if core else None}
"core_sha256": core.sha256 if core else None,
"record_version": mr.RECORD_VERSION, "oracle_sha256": self.oracle_sha}
if result.reason:
entry["reason"] = result.reason
if result.log:
@@ -1042,6 +1068,20 @@ class Campaign:
self.state["checks"].setdefault(emulator, {})[key] = {
"status": PENDING, "reason": reason, "at": now_iso(), "profile_digest": digest}
def needed(self, emulator: str, key: str, core: Core, digest: str) -> bool:
return should_run(self.state["checks"].get(emulator, {}).get(key), core.sha256, digest,
self.oracle_sha)
def check(self, emulator: str, key: str, core: Core, digest: str, argv: list[str],
log_name: str) -> dict | None:
"""The record of a check: run now when needed, else the one kept."""
if not self.needed(emulator, key, core, digest):
previous = self.state["checks"].get(emulator, {}).get(key) or {}
return self.read_record(emulator, previous.get("record"))
result = self.probe(argv, log_name)
self.remember(emulator, key, result, core, digest)
return result.record
def system_entries(self, profile: dict, system: str) -> list[tuple[int, dict]]:
from validation import filter_files_by_mode
@@ -1060,84 +1100,148 @@ class Campaign:
held.sort(key=lambda item: (item[1].get("priority", 1 << 30), item[0]))
return held[0] if held else None
@staticmethod
def other_regions(entry: dict, staged: dict[str, tuple[int, dict, Path]], local: Path) -> list[Path]:
"""Held dumps of the same system for regions this entry does not
serve, one per distinct file: the wrong-region variants."""
mine = set(regions_of(entry))
if not mine or "world" in mine:
return []
out: dict[str, Path] = {}
for _, other, path in staged.values():
theirs = set(regions_of(other))
if theirs and "world" not in theirs and not theirs & mine and path.resolve() != local.resolve():
out.setdefault(file_sha1(path), path)
return sorted(out.values(), key=lambda p: p.name)[:MAX_REGION_DUMPS]
def run_unit_content(self, unit: Unit, core: Core, content: Content, digest: str) -> None:
emulator, profile = unit.emulator, unit.profile
checks = self.state["checks"].setdefault(emulator, {})
entries = self.system_entries(profile, content.system)
boot = self.boot_entry(entries, content)
keys = {name: self.check_key(core, content, name) for name in ("files", "bench")}
if boot:
keys["boot"] = self.check_key(core, content, f"boot:{boot[1]['name']}")
todo = [k for k in keys.values() if should_run(checks.get(k), core.sha256, digest)]
if not todo:
return
try:
folder = self.store.get(content)
load = load_file(folder, content.load)
except Unavailable as exc:
for key in todo:
self.pending(emulator, key, str(exc), digest)
return
frames = ["--frames", str(content.frames), "--frame-hash-every", str(content.frame_hash_every)]
stem = f"{core.name}-{slug(content.name)}"
# files: every declared file the collection holds, at its place.
staged: dict[str, tuple[int, dict, Path]] = {}
for index, entry in entries:
local = self.local_file(entry)
if local and entry_dest(entry) not in staged:
staged[entry_dest(entry)] = (index, entry, local)
files_record = None
if keys["files"] in todo:
argv = self.probe_args("run", core, emulator, slug(f"files-{stem}"), content, load) + frames
for dest, (_, _, local) in sorted(staged.items()):
argv += ["--system-file", f"{dest}={local}"]
result = self.probe(argv, f"{emulator}-{stem}-files")
self.remember(emulator, keys["files"], result, core, digest)
files_record = result.record
elif checks.get(keys["files"], {}).get("status") == DONE:
files_record = self.read_record(emulator, checks[keys["files"]].get("record"))
tested = {dest: item for dest, item in staged.items() if region_matches(item[1], content.region)}
keys = {"files": self.check_key(core, content, "files"),
"none": self.check_key(core, content, "none")}
for dest in tested:
keys[f"negative:{dest}"] = self.check_key(core, content, f"negative:{dest}")
if self.settings.bench:
keys["bench"] = self.check_key(core, content, "bench")
if not any(self.needed(emulator, key, core, digest) for key in keys.values()):
return
try:
folder = self.store.get(content)
load = load_file(folder, content.load)
except Unavailable as exc:
for key in keys.values():
if self.needed(emulator, key, core, digest):
self.pending(emulator, key, str(exc), digest)
return
frames = ["--frames", str(content.frames), "--frame-hash-every", str(content.frame_hash_every)]
repeat = ["--repeat", str(self.settings.check_repeat)]
stem = f"{core.name}-{slug(content.name)}"
boot_record = None
if boot and keys["boot"] in todo:
index, entry = boot
local = self.local_file(entry)
dest = entry_dest(entry)
argv = self.probe_args("check", core, emulator, slug(f"boot-{entry['name']}-{stem}"),
content, load) + frames
argv += ["--vary", f"{dest}=", "--vary", f"{dest}={local}", "--observable", "frame_hashes",
"--repeat", str(self.settings.check_repeat),
"--claim", f"{core.name} reaches the game on {content.name} without {entry['name']}",
"--method", (f"Two variants, {entry['name']} absent and {entry['name']} as the "
f"collection holds it, each run {self.settings.check_repeat} times "
f"from a fresh process for {content.frames} frames with no button "
f"pressed. Every {content.frame_hash_every}th frame is hashed. The "
f"runs without the file are compared with the runs with it on the "
f"frames that are not one colour.")]
result = self.probe(argv, f"{emulator}-{stem}-boot")
self.remember(emulator, keys["boot"], result, core, digest)
boot_record = result.record
if boot_record:
absent, present = (v["label"] for v in boot_record["variants"])
verdicts = rules.rule_bios_need(rules.boot_outcome(boot_record, absent, present),
entry, boot_record["id"])
# files: every declared file the collection holds, at its place.
argv = self.probe_args("run", core, emulator, slug(f"files-{stem}"), content, load) + frames + repeat
for dest, (_, _, local) in sorted(staged.items()):
argv += ["--system-file", f"{dest}={local}"]
files_record = self.check(emulator, keys["files"], core, digest, argv, f"{emulator}-{stem}-files")
# none: no BIOS at all.
argv = self.probe_args("run", core, emulator, slug(f"none-{stem}"), content, load) + frames + repeat
none_record = self.check(emulator, keys["none"], core, digest, argv, f"{emulator}-{stem}-none")
if not files_record or not none_record:
return
full = rules.run_of(files_record, files_record["variants"][0]["label"])
none = rules.run_of(none_record, none_record["variants"][0]["label"])
self.log(f"{emulator}: {core.name} on {content.name}: full set {full.verdict}, "
f"no BIOS {none.verdict}")
for dest, (index, entry, local) in sorted(tested.items()):
record = self.negative(unit, core, content, load, frames, digest, keys[f"negative:{dest}"],
dest, entry, local, staged)
if not record:
continue
labels = [v["label"] for v in record["variants"]]
present = rules.run_of(record, labels[0])
zeros, half = (rules.run_of(record, label) for label in labels[1:3])
for label in labels[3:]:
self.log(f"{emulator}: {rules.rule_region(rules.run_of(record, label), present, entry, content.region)}")
if dest in self.opened(files_record, 0) or dest in self.opened(record, 0):
verdicts = rules.rule_need(none, present, entry)
validation = rules.rule_validation(zeros, half, none, present, entry)
if validation:
verdicts.append(validation)
self.settle(emulator, content, core, index, entry, verdicts)
self.follow_aliases(unit, core, content, load, frames, digest, index, entry,
local, boot_record, absent)
variants = self.variant_files(entry, local)
if variants:
self.run_variants(unit, core, content, load, frames, digest, index, entry, dest,
local, variants)
else:
self.moved(unit, core, content, load, frames, digest, index, entry, dest, local,
files_record, record)
if files_record and keys["files"] in todo:
self.read_files(unit, core, content, load, frames, digest, staged, files_record)
boot = self.boot_entry(entries, content)
if boot and entry_dest(boot[1]) in tested:
index, entry = boot
self.follow_aliases(unit, core, content, load, frames, digest, index, entry,
tested[entry_dest(entry)][2], none_record)
if keys["bench"] in todo:
if self.settings.bench and full.verdict == rules.BOOTS:
argv = self.probe_args("bench", core, emulator, slug(f"bench-{stem}"), content, load)
argv += ["--warmup", str(self.settings.bench_warmup), "--frames",
str(self.settings.bench_frames), "--repeat", str(self.settings.bench_repeat),
"--no-trace"]
if boot:
local = self.local_file(boot[1])
argv += ["--system-file", f"{entry_dest(boot[1])}={local}"]
result = self.probe(argv, f"{emulator}-{stem}-bench")
self.remember(emulator, keys["bench"], result, core, digest)
argv += ["--system-file", f"{entry_dest(boot[1])}={self.local_file(boot[1])}"]
self.check(emulator, keys["bench"], core, digest, argv, f"{emulator}-{stem}-bench")
def negative(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
digest: str, key: str, dest: str, entry: dict, local: Path,
staged: dict[str, tuple[int, dict, Path]]) -> dict | None:
"""The file alone, then wrong files under its name: zeros of its
size, its first half, and other regions' dumps."""
stem = f"{core.name}-{slug(content.name)}"
size = local.stat().st_size
others = self.other_regions(entry, staged, local)
argv = self.probe_args("check", core, unit.emulator, slug(f"negative-{entry['name']}-{stem}"),
content, load) + frames
argv += ["--vary", f"{dest}={local}", "--vary", f"{dest}=@zeros:{size}",
"--vary", f"{dest}=@head:{size // 2}:{local}"]
for other in others:
argv += ["--vary", f"{dest}={other}"]
regions = f", then {', '.join(p.name for p in others)} staged under its name" if others else ""
argv += ["--observable", "frame_hashes", "--repeat", str(self.settings.check_repeat),
"--claim", (f"{core.name} boots {content.name} with {entry['name']} alone and "
f"refuses a wrong file under its name"),
"--method", (f"{entry['name']} alone as {dest}, then {size} zero bytes and its "
f"first {size // 2} bytes{regions}, nothing else staged. Each run "
f"{self.settings.check_repeat} times from a fresh process for "
f"{content.frames} frames with no button pressed, every "
f"{content.frame_hash_every}th frame hashed. The boot oracle judges "
f"each variant against the reference frames of {content.name}.")]
return self.check(unit.emulator, key, core, digest, argv, f"{unit.emulator}-{stem}-negative")
def moved(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
digest: str, index: int, entry: dict, dest: str, local: Path, files_record: dict,
alone: dict) -> None:
"""A file the core opened neither in the full set nor alone: where
else it looked for a file of that name, else flagged."""
probed = [p for record in (files_record, alone) for r in record["variants"][0]["repetitions"]
for p in r.get("files_probed_absent") or ()]
others = [p for p in dict.fromkeys(probed) if p != dest
and p.rsplit("/", 1)[-1].casefold() == dest.rsplit("/", 1)[-1].casefold()]
for candidate in others[:MAX_ALIAS_CANDIDATES]:
found = self.follow_up(unit, core, content, load, frames, digest, entry, local, candidate,
f"path:{entry['name']}:{candidate}")
if found and candidate in self.opened(found, 0):
verdict = rules.rule_path(entry, dest, rules.Opened(candidate, found["id"]))
if verdict:
self.settle(unit.emulator, content, core, index, entry, [verdict])
return
verdict = rules.rule_never_opened(entry, [files_record["id"], alone["id"]], content.name)
self.settle(unit.emulator, content, core, index, entry, [verdict])
def read_record(self, emulator: str, rid: str | None) -> dict | None:
if not rid:
@@ -1150,49 +1254,10 @@ class Campaign:
return json.loads(line)
return None
def read_files(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
digest: str, staged: dict[str, tuple[int, dict, Path]], record: dict) -> None:
"""Which staged files the core opened: path follow-ups, variants, flags."""
emulator = unit.emulator
rep = record["variants"][0]["repetitions"][0]
if rep.get("files_opened") is None:
return
opened = {item["path"] for item in rep.get("files_opened") or ()}
probed = list(rep.get("files_probed_absent") or ())
for dest, (index, entry, local) in sorted(staged.items()):
if dest in opened:
variants = self.variant_files(entry, local)
if variants:
self.run_variants(unit, core, content, load, frames, digest, index, entry, dest,
local, variants)
continue
if not region_matches(entry, content.region):
continue
# A fallback is read only when the files before it are absent:
# staged alone, it shows whether the core reads it at all.
alone = self.follow_up(unit, core, content, load, frames, digest, entry, local, dest,
f"alone:{entry['name']}")
if alone is None:
continue
if dest in self.opened(alone, 1):
continue
probed_alone = [p for r in alone["variants"][0]["repetitions"]
for p in r.get("files_probed_absent") or ()]
others = [p for p in dict.fromkeys(probed + probed_alone) if p != dest
and p.rsplit("/", 1)[-1].casefold() == dest.rsplit("/", 1)[-1].casefold()]
moved = False
for candidate in others[:MAX_ALIAS_CANDIDATES]:
record_c = self.follow_up(unit, core, content, load, frames, digest, entry, local,
candidate, f"path:{entry['name']}:{candidate}")
if record_c and candidate in self.opened(record_c, 1):
verdict = rules.rule_path(entry, dest, rules.Opened(candidate, record_c["id"]))
if verdict:
self.settle(emulator, content, core, index, entry, [verdict])
moved = True
break
if not moved:
verdict = rules.rule_never_opened(entry, [record["id"], alone["id"]], content.name)
self.settle(emulator, content, core, index, entry, [verdict])
def judged(self, emulator: str, rid: str) -> bool:
"""True for a record the boot oracle judged: only those prove a field."""
record = self.read_record(emulator, rid)
return bool(record) and record.get("record_version", 0) >= 4
@staticmethod
def opened(record: dict, position: int) -> set[str]:
@@ -1203,34 +1268,23 @@ class Campaign:
def follow_up(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
digest: str, entry: dict, local: Path, dest: str, check: str) -> dict | None:
"""The file staged at dest alone, against nothing there."""
key = self.check_key(core, content, check)
previous = self.state["checks"].get(unit.emulator, {}).get(key)
if not should_run(previous, core.sha256, digest):
return self.read_record(unit.emulator, (previous or {}).get("record"))
"""The file staged at dest alone: does the core open it there."""
stem = f"{core.name}-{slug(content.name)}"
argv = self.probe_args("check", core, unit.emulator, slug(f"{check}-{stem}"), content, load)
argv += frames + ["--vary", f"{dest}=", "--vary", f"{dest}={local}",
"--observable", "frame_hashes", "--repeat", "1",
"--claim", f"{core.name} opens {entry['name']} as {dest}",
"--method", (f"{entry['name']} staged as {dest}, against nothing there, "
f"{content.frames} frames of {content.name}. The files the "
f"core opened are read from strace.")]
result = self.probe(argv, f"{unit.emulator}-{stem}-{slug(check)}")
self.remember(unit.emulator, key, result, core, digest)
return result.record
argv = self.probe_args("run", core, unit.emulator, slug(f"{check}-{stem}"), content, load)
argv += frames + ["--system-file", f"{dest}={local}"]
return self.check(unit.emulator, self.check_key(core, content, check), core, digest, argv,
f"{unit.emulator}-{stem}-{slug(check)}")
def follow_aliases(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
digest: str, index: int, entry: dict, local: Path, record: dict,
absent_label: str) -> None:
"""Names the core looked for, with the file absent, that no entry declares."""
digest: str, index: int, entry: dict, local: Path, none_record: dict) -> None:
"""Names the core looked for, with no BIOS, that no entry declares."""
declared = {e["name"] for e in unit.profile.get("files") or ()}
declared |= {a for e in unit.profile.get("files") or () for a in e.get("aliases") or ()}
suffix = Path(entry["name"]).suffix.casefold()
dest = entry_dest(entry)
folder = dest.rsplit("/", 1)[0] if "/" in dest else ""
probed: list[str] = []
for rep in rules.variant(record, absent_label)["repetitions"]:
for rep in none_record["variants"][0]["repetitions"]:
for path in rep.get("files_probed_absent") or ():
name = path.rsplit("/", 1)[-1]
where = path.rsplit("/", 1)[0] if "/" in path else ""
@@ -1240,7 +1294,7 @@ class Campaign:
for candidate in probed[:MAX_ALIAS_CANDIDATES]:
found = self.follow_up(unit, core, content, load, frames, digest, entry, local, candidate,
f"alias:{entry['name']}:{candidate.rsplit('/', 1)[-1]}")
if found and candidate in self.opened(found, 1):
if found and candidate in self.opened(found, 0):
verdict = rules.rule_alias(self.reload_entry(unit, index), rules.Opened(candidate, found["id"]))
if verdict:
self.settle(unit.emulator, content, core, index, self.reload_entry(unit, index), [verdict])
@@ -1249,7 +1303,7 @@ class Campaign:
digest: str, index: int, entry: dict, dest: str, primary: Path,
variants: list[Path]) -> None:
key = self.check_key(core, content, f"variants:{entry['name']}")
if not should_run(self.state["checks"].get(unit.emulator, {}).get(key), core.sha256, digest):
if not self.needed(unit.emulator, key, core, digest):
return
stem = f"{core.name}-{slug(content.name)}"
argv = self.probe_args("check", core, unit.emulator, slug(f"variants-{entry['name']}-{stem}"),
@@ -1257,16 +1311,14 @@ class Campaign:
for path in [primary, *variants]:
argv += ["--vary", f"{dest}={path}"]
argv += ["--observable", "frame_hashes", "--repeat", str(self.settings.check_repeat),
"--claim", f"the copies of {entry['name']} change what {core.name} shows on {content.name}",
"--method", (f"Each copy the collection holds staged as {dest}, each run "
f"{self.settings.check_repeat} times for {content.frames} frames, "
f"compared on the sampled frame hashes.")]
result = self.probe(argv, f"{unit.emulator}-{stem}-variants")
self.remember(unit.emulator, key, result, core, digest)
if result.record:
status = result.record["check"]["result"]["status"]
verdict = rules.rule_variants(status, [v["label"] for v in result.record["variants"]],
entry, result.record["id"])
"--claim", f"every copy of {entry['name']} boots {content.name} on {core.name}",
"--method", (f"Each copy the collection holds staged alone as {dest}, each run "
f"{self.settings.check_repeat} times for {content.frames} frames. The "
f"boot oracle judges each copy.")]
record = self.check(unit.emulator, key, core, digest, argv, f"{unit.emulator}-{stem}-variants")
if record:
verdict = rules.rule_variants([rules.run_of(record, v["label"]) for v in record["variants"]],
entry)
if verdict:
self.settle(unit.emulator, content, core, index, entry, [verdict])
@@ -1291,9 +1343,10 @@ class Campaign:
agrees = verdict.field in rules.SETTLED_FIELDS and rules.agrees(fresh, verdict)
if verdict.apply:
updates: dict[str, object] = {}
if not agrees or verdict.field not in fresh:
if not agrees or (verdict.field not in fresh and verdict.field != "validation"):
updates[verdict.field] = verdict.value
evidence = rules.merge_evidence(fresh.get("evidence"), verdict)
evidence = rules.merge_evidence(fresh.get("evidence"), verdict,
keep=lambda rid: self.judged(emulator, rid))
if evidence != fresh.get("evidence"):
updates["evidence"] = evidence
if updates:
@@ -1364,6 +1417,60 @@ class Campaign:
return record["core"].get("library_version")
return None
# -- reference frames -----------------------------------------------------------
def references(self, only: set[str], system: str | None, png_dir: Path) -> list[dict]:
"""The full set run once per core and content, frames written as PNG,
and the screens that stay on long enough to serve as references."""
self.deadline = self.host.clock() + self.settings.budget
listed = []
for unit in self.units(only, system):
contents = self.unit_contents(unit, system)
if unit.pending or not contents:
continue
try:
cores = self.unit_cores(unit.emulator, unit.profile, unit.buildbot_only)
except Unavailable as exc:
self.log(f"{unit.emulator}: {exc}")
continue
for core in cores:
for content in contents:
self.guard()
self.store.hold(content, "references")
try:
load = load_file(self.store.get(content), content.load)
except Unavailable as exc:
self.log(f"{unit.emulator}: {exc}")
continue
stem = f"{core.name}-{slug(content.name)}"
target = png_dir / slug(core.name) / slug(content.name)
shutil.rmtree(target, ignore_errors=True)
argv = self.probe_args("run", core, unit.emulator, slug(f"references-{stem}"),
content, load)
argv += ["--frames", str(content.frames), "--frame-hash-every",
str(content.frame_hash_every), "--repeat", str(self.settings.check_repeat),
"--png-dir", str(target)]
for index, entry in self.system_entries(unit.profile, content.system):
local = self.local_file(entry)
if local and f"{entry_dest(entry)}=" not in " ".join(argv):
argv += ["--system-file", f"{entry_dest(entry)}={local}"]
result = self.probe(argv, f"{unit.emulator}-{stem}-references")
self.store.release(content, "references")
if not result.record:
self.log(f"{unit.emulator}: references of {content.name}: {result.reason}")
continue
record = result.record
listed.append({
"emulator": unit.emulator, "core": core.name,
"library_name": record["core"]["library_name"], "content": content.name,
"content_sha1": record["inputs"]["content"]["sha1"], "record": record["id"],
"verdict": record["variants"][0]["verdict"],
"png_dir": str(target / record["variants"][0]["label"]),
"candidates": reference_candidates(record),
})
save_state(self.paths.state, self.state)
return listed
# -- the pass -----------------------------------------------------------------
def units(self, only: set[str], system: str | None) -> list[Unit]:
@@ -1567,6 +1674,32 @@ class Campaign:
return made
def reference_candidates(record: dict, min_samples: int = 3) -> list[dict]:
"""Screens of a record's first variant that are not one colour and stay
on for at least min_samples consecutive samples in every repetition:
what a reviewer looks at to choose reference frames."""
found: list[dict[str, tuple[int, int]]] = []
for rep in record["variants"][0]["repetitions"]:
observables = rep.get("observables") or {}
uniform = set(observables.get("uniform_frames") or ())
runs: dict[str, tuple[int, int]] = {}
previous, start, length = None, 0, 0
for frame, digest in [*(observables.get("frame_hashes") or ()), (None, None)]:
if digest == previous:
length += 1
continue
if previous and previous not in uniform and length >= min_samples:
first, longest = runs.get(previous, (start, 0))
runs[previous] = (min(first, start), max(longest, length))
previous, start, length = digest, frame, 1
found.append(runs)
common = set(found[0]) if found else set()
for runs in found[1:]:
common &= set(runs)
return sorted(({"sha1": digest, "first_frame": found[0][digest][0], "samples": found[0][digest][1]}
for digest in common), key=lambda item: item["first_frame"])
def reason_class(reason: str) -> str:
"""A pending reason without its detail, to count them."""
head = reason.split(":", 1)[0].strip() if ":" in reason else reason
@@ -1661,9 +1794,15 @@ def main(argv: Sequence[str] | None = None) -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0],
formatter_class=argparse.RawDescriptionHelpFormatter)
sub = parser.add_subparsers(dest="command", required=True)
run_p = sub.add_parser("run", help="one pass: cores, content, checks, rules")
run_p.add_argument("--only", default="", help="emulator profiles or core names, comma-separated")
run_p.add_argument("--system", help="one system id")
scope = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
scope.add_argument("--only", default="", help="emulator profiles or core names, comma-separated")
scope.add_argument("--system", help="one system id")
scope.add_argument("--romfetch", default="",
help=f"romfetch command (default: ${ROMFETCH_ENV}, else romfetch on PATH)")
scope.add_argument("--offline", action="store_true", help="no buildbot, no romfetch, no build")
scope.add_argument("--allow-unsandboxed", action="store_true",
help="run cores and builds without bubblewrap when it is missing (refused by default)")
run_p = sub.add_parser("run", parents=[scope], help="one pass: cores, content, checks, rules")
run_p.add_argument("--budget", type=parse_budget, default=6 * 3600.0,
help="stop starting checks after this long (6h, 90m, 3600s)")
run_p.add_argument("--max-load-wait", type=float, default=3600,
@@ -1671,34 +1810,28 @@ def main(argv: Sequence[str] | None = None) -> int:
run_p.add_argument("--min-free-gb", type=float, default=30, help="pause below this free space")
run_p.add_argument("--cache-cap-gb", type=float, default=20,
help="content the campaign fetched is evicted past this size, oldest use first")
run_p.add_argument("--romfetch", default="",
help=f"romfetch command (default: ${ROMFETCH_ENV}, else romfetch on PATH)")
run_p.add_argument("--offline", action="store_true", help="no buildbot, no romfetch, no build")
run_p.add_argument("--allow-unsandboxed", action="store_true",
help="run cores and builds without bubblewrap when it is missing (refused by default)")
run_p.add_argument("--bench", action="store_true",
help="also time each content that boots (benchmarks only choose between variants)")
run_p.add_argument("--dry-run", action="store_true", help="print the plan, run nothing")
run_p.add_argument("--commit", action="store_true",
help="commit records, findings and each changed profile behind the gate")
run_p.add_argument("--commit-every", type=int, default=0, metavar="N",
help="with --commit, also commit after every N emulators")
refs_p = sub.add_parser("references", parents=[scope],
help="run the full set with PNG frames and list reference candidates")
refs_p.add_argument("--png-dir", default=str(ROOT / "tmp" / "campaign" / "references"),
help="where the frames are written, one folder per core and content")
status_p = sub.add_parser("status", help="done, failed and pending checks, and why")
status_p.add_argument("--json", action="store_true", help="the report as JSON")
findings_p = sub.add_parser("findings", help="profile claims the runs contradict, open for review")
findings_p.add_argument("--all", action="store_true", help="applied and resolved too")
findings_p.add_argument("--json", action="store_true", help="the findings as JSON")
sub.add_parser("gate", help="tests, schemas, verify and pack of the changed profiles")
timer_p = sub.add_parser("install-timer",
help="write the autorun units, the campaign job's timer among them")
timer_p.add_argument("--enable", action="store_true",
help="also reload systemd and enable the timers")
args = parser.parse_args(argv)
paths = Paths.default()
if args.command == "gate":
return gate(paths.root, changed_profiles(paths.root))
if args.command == "install-timer":
command = [sys.executable, str(ROOT / "scripts" / "autorun.py"), "install-timers"]
return subprocess.run(command + (["--enable"] if args.enable else []), check=False).returncode
if args.command == "findings":
lines = open_findings(paths.root, args.all)
if args.json:
@@ -1713,6 +1846,9 @@ def main(argv: Sequence[str] | None = None) -> int:
settings = Settings(budget=args.budget, max_load_wait=args.max_load_wait,
min_free_gb=args.min_free_gb, cache_cap_gb=args.cache_cap_gb,
romfetch=args.romfetch, network=not args.offline,
allow_unsandboxed=args.allow_unsandboxed, bench=args.bench)
elif args.command == "references":
settings = Settings(romfetch=args.romfetch, network=not args.offline,
allow_unsandboxed=args.allow_unsandboxed)
campaign = Campaign(paths, settings)
if args.command == "status":
@@ -1720,13 +1856,22 @@ def main(argv: Sequence[str] | None = None) -> int:
print(json.dumps(report, indent=2) if args.json else render_status(report))
return 0
only = {item.strip() for item in args.only.split(",") if item.strip()}
if args.command == "references":
try:
with host_lock(campaign.lock_path):
listed = campaign.references(only, args.system, Path(args.png_dir))
except autorun.LockBusy as exc:
print(f"campaign: deferred, {exc}", file=sys.stderr)
return 75
print(json.dumps(listed, indent=2))
return 0
if args.dry_run:
campaign.run(only, args.system, dry_run=True)
return 0
campaign.dirty_at_start = set(autorun.changed_files(paths.root)) if args.commit else set()
def stop(signum: int, frame: object) -> None:
# A timer's TimeoutStartSec ends the pass like Ctrl-C: state saved, pass line written.
# SIGTERM ends the pass like Ctrl-C: state saved, pass line written.
raise KeyboardInterrupt
signal.signal(signal.SIGTERM, stop)
try:
scripts/campaign_profiles.py
+1
View File
@@ -23,6 +23,7 @@ ANCHORS = {
"hle_fallback": ("required", "name"),
"path": ("name",),
"aliases": ("path", "name"),
"validation": ("sha1", "md5", "crc32", "size", "name"),
}
scripts/campaign_rules.py
+168 -97
View File
@@ -1,25 +1,36 @@
"""The campaign's fixed rule table: which observable settles which profile field.
"""The campaign's fixed rule table: which verdicts settle which profile field.
Each rule is a pure function of a core probe record (or what was read from
one) and a profile entry. It returns verdicts. A verdict with `apply` set is
settled by the table and written into the profile with the record ids as
`evidence:`. Any other verdict goes to the review queue
(measurements/findings.jsonl).
Each rule is a pure function of verdicts read from core probe records (the
boot oracle of core_probe.py: boots, bios_error, crash, hang, unknown) and
a profile entry. It returns verdicts on profile fields. One with `apply` set
is settled by the table and written into the profile with the record ids as
`evidence:`. Any other goes to the review queue (measurements/findings.jsonl).
observable field settled
boots to the game with no file hle_fallback true
no game screen without it, the game with it required true
hle_fallback false
opens the staged file under another path path that path
opens the staged file under another name aliases name added
a staged file the content should use, never read - flagged only
Three runs are compared for a file, all on the same content: `none` with no
BIOS at all, `present` with the file alone, and wrong files staged under its
name (zeros of its size, its first half, another region's dump).
"Boots to the game" is read from the sampled frames: the run without the
file shows at least MIN_SHARED_SCREENS distinct screens that the run with
the file also shows, single-colour frames excluded. "No game screen" is a
run that did not load, or whose every sampled frame is a single colour. A
run that shows screens of its own that the reference never shows settles
nothing.
runs field settled
none boots hle_fallback true
none fails, present boots required true
hle_fallback false
zeros refused, present boots validation a content check
zeros used, present boots validation no content check
half refused, zeros used validation size
half used validation no size check
opens the staged file under another path path that path
opens the staged file under another name aliases name added
a staged file the content should use, never - flagged only
read
copies of one file that boot and fail - queued
"Fails" is bios_error, crash or hang. A wrong file is refused when the
core reports a BIOS error, or when it boots without the BIOS screens the
right file shows (it fell back to a built-in BIOS). It is used when it
turns the run into a hang or a crash that the run without any file does not
show. Anything else settles nothing. Of validation, the table writes only a
missing `size`. A contradicted content check goes to review: which hash a
core computes is read in its code.
Standard library only.
"""
@@ -29,23 +40,34 @@ from __future__ import annotations
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
MIN_SHARED_SCREENS = 2
BOOTS = "boots"
FAILS = "fails"
INCONCLUSIVE = "inconclusive"
BIOS_ERROR = "bios_error"
CRASH = "crash"
HANG = "hang"
UNKNOWN = "unknown"
FAILS = (BIOS_ERROR, CRASH, HANG)
REFUSED = "refused"
USED = "used"
CONTENT_CHECKS = ("md5", "sha1", "sha256", "crc32", "adler32")
# A field an entry leaves out reads as this value.
FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": []}
SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases")
FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": [],
"validation": []}
SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases", "validation")
@dataclass(frozen=True)
class Boot:
"""What the runs without the file showed, against the runs with it."""
class Run:
"""The verdict of one variant of a record: what a rule reads."""
status: str
detail: str
shared: int = 0
verdict: str
record_id: str
label: str
stages: tuple[str, ...] = ()
@property
def shows_bios(self) -> bool:
return "bios" in self.stages
@dataclass(frozen=True)
@@ -67,13 +89,6 @@ class Opened:
extra: dict = field(default_factory=dict)
def screens(repetition: dict) -> set[str]:
"""The distinct sampled frames of a repetition that are not one colour."""
observables = repetition.get("observables") or {}
uniform = set(observables.get("uniform_frames") or ())
return {digest for _, digest in observables.get("frame_hashes") or () if digest not in uniform}
def variant(record: dict, label: str) -> dict:
for item in record.get("variants") or ():
if item.get("label") == label:
@@ -81,43 +96,16 @@ def variant(record: dict, label: str) -> dict:
raise KeyError(f"record {record.get('id')}: no variant {label!r}")
def boot_outcome(record: dict, absent_label: str, present_label: str) -> Boot:
"""Whether the core reached the game without the file.
def run_of(record: dict, label: str) -> Run:
"""A variant's verdict as a rule reads it. A record from before the
oracle has none, and reads as unknown."""
item = variant(record, label)
evidence = item.get("verdict_evidence") or {}
return Run(item.get("verdict", UNKNOWN), record["id"], label, tuple(evidence.get("stages") or ()))
Every repetition must agree: one that boots and one that does not leave
the question open.
"""
present = variant(record, present_label)["repetitions"]
if any(not rep.get("loaded") for rep in present):
return Boot(INCONCLUSIVE, "with the file, the core did not load the content")
reference: set[str] = set()
for rep in present:
shown = screens(rep)
if not shown:
return Boot(INCONCLUSIVE, "with the file, the core showed no game screen")
reference |= shown
outcomes: list[str] = []
shared_counts: list[int] = []
for rep in variant(record, absent_label)["repetitions"]:
if not rep.get("loaded"):
outcomes.append(FAILS)
continue
shown = screens(rep)
if not shown:
outcomes.append(FAILS)
continue
shared = len(shown & reference)
shared_counts.append(shared)
outcomes.append(BOOTS if shared >= MIN_SHARED_SCREENS else INCONCLUSIVE)
shared = min(shared_counts) if shared_counts else 0
if all(o == BOOTS for o in outcomes):
return Boot(BOOTS, f"without the file, every run showed {shared} or more screens the "
f"run with the file shows", shared)
if all(o == FAILS for o in outcomes):
return Boot(FAILS, "without the file, no run loaded the content or showed more than "
"one colour, while the run with the file reached the game")
return Boot(INCONCLUSIVE, "without the file, the runs showed screens the run with the file "
"never shows, or disagreed with each other", shared)
def ids(*runs: Run) -> tuple[str, ...]:
return tuple(dict.fromkeys(run.record_id for run in runs))
def current(entry: dict, name: str) -> object:
@@ -130,32 +118,110 @@ def agrees(entry: dict, verdict: Verdict) -> bool:
"""True when the profile already says what the run showed."""
if verdict.field == "aliases":
return set(verdict.value or ()) <= set(entry.get("aliases") or ())
if verdict.field == "validation":
return verdict.rule == "validation_agrees"
return current(entry, verdict.field) == verdict.value
def rule_bios_need(boot: Boot, entry: dict, record_id: str) -> list[Verdict]:
"""Boot with no file against boot with the declared file."""
def rule_need(none: Run, present: Run, entry: dict) -> list[Verdict]:
"""No BIOS at all against the file alone: required and hle_fallback."""
name = entry["name"]
evidence = (record_id,)
if boot.status == BOOTS:
if none.verdict == BOOTS:
out = [Verdict("boots_without_file", "hle_fallback", True,
f"the core reaches the game without {name}: {boot.detail}", evidence, True)]
f"the core boots the content with no BIOS at all, {name} included",
ids(none), True)]
if entry.get("required") is True:
out.append(Verdict(
"boots_without_file", "required", False,
f"the profile says {name} is required, and the core reaches the game "
f"without it. The table settles hle_fallback from this run, not required",
evidence))
f"the profile says {name} is required, and the core boots without it. The table "
f"settles hle_fallback from this run, not required", ids(none)))
return out
if boot.status == FAILS:
if none.verdict in FAILS and present.verdict == BOOTS:
return [
Verdict("fails_without_file", "required", True,
f"the core reaches the game only with {name}: {boot.detail}", evidence, True),
f"with no BIOS the run ends in {none.verdict}, with {name} alone it boots",
ids(none, present), True),
Verdict("fails_without_file", "hle_fallback", False,
f"no stand-in for {name}: {boot.detail}", evidence, True),
f"no stand-in for {name}: with no BIOS the run ends in {none.verdict}",
ids(none, present), True),
]
return [Verdict("inconclusive", "hle_fallback", None,
f"the runs with and without {name} settle nothing: {boot.detail}", evidence)]
return [Verdict("unsettled", "hle_fallback", None,
f"with no BIOS the run is {none.verdict} and with {name} alone {present.verdict}: "
f"no rule settles a field from that", ids(none, present))]
def file_use(wrong: Run, none: Run, present: Run) -> str | None:
"""Whether the core refused a wrong file or ran it, or None."""
if present.verdict != BOOTS:
return None
if wrong.verdict == BIOS_ERROR:
return REFUSED
if wrong.verdict == BOOTS:
if wrong.shows_bios:
return USED
return REFUSED if present.shows_bios else None
if wrong.verdict in (HANG, CRASH) and none.verdict in (BOOTS, BIOS_ERROR):
return USED
return None
def declared_checks(entry: dict) -> tuple[set[str], bool]:
"""The checks the entry's validation names, and whether it is one list
(a per-build dict is never edited by the table)."""
value = entry.get("validation") or []
if isinstance(value, dict):
return {str(v) for items in value.values() for v in items or ()}, False
return {str(v) for v in value}, True
def rule_validation(zeros: Run | None, half: Run | None, none: Run, present: Run,
entry: dict) -> Verdict | None:
"""Same-size and short wrong files against the right one: validation."""
name = entry["name"]
content = {REFUSED: True, USED: False}.get(file_use(zeros, none, present)) if zeros else None
half_use = file_use(half, none, present) if half else None
size: bool | None = None
if half_use == USED:
size, content = False, False if content is None else content
elif half_use == REFUSED and content is False:
size = True
if content is None and size is None:
return None
declared, editable = declared_checks(entry)
evidence = ids(*(run for run in (none, present, zeros, half) if run))
said = sorted(declared)
shown = []
problems = []
if content is not None:
shown.append("checks the content" if content else "does not check the content")
if content != bool(declared & set(CONTENT_CHECKS)):
problems.append("a same-size file of zeros is " + ("refused" if content else "run")
+ f", the profile declares validation {said}")
missing_size = size is True and "size" not in declared
if size is not None:
shown.append("checks the size" if size else "does not check the size")
if size is False and "size" in declared:
problems.append(f"the first half of {name} is run, the profile declares a size check")
statement = f"the core {' and '.join(shown)} of {name}"
if problems:
return Verdict("validation_contradicted", "validation", said,
f"{statement}: {'; '.join(problems)}", evidence)
if missing_size:
if not editable:
return Verdict("validation_contradicted", "validation", said,
f"{statement}: the per-build validation has no size check", evidence)
value = [*(entry.get("validation") or []), "size"]
return Verdict("validation_size_added", "validation", value,
f"{statement}: half of the file is refused while zeros of its size are run",
evidence, True)
return Verdict("validation_agrees", "validation", entry.get("validation") or [], statement,
evidence, editable)
def rule_region(dump: Run, present: Run, entry: dict, game_region: str | None) -> str:
"""What the core did with another region's dump under this file's name."""
return (f"{dump.label} staged as {entry['name']} with a {game_region or 'region-less'} game: "
f"{dump.verdict}, {entry['name']} itself: {present.verdict}")
def rule_path(entry: dict, declared: str, opened: Opened) -> Verdict | None:
@@ -184,20 +250,25 @@ def rule_never_opened(entry: dict, record_ids: Sequence[str], content: str) -> V
f"{content}", tuple(record_ids))
def rule_variants(status: str, labels: Iterable[str], entry: dict, record_id: str) -> Verdict | None:
"""Copies of one file that change what the core shows: a pack choice to review."""
if status != "differs":
def rule_variants(runs: Iterable[Run], entry: dict) -> Verdict | None:
"""Copies of one file whose verdicts differ: a pack choice to review."""
runs = list(runs)
if len({run.verdict for run in runs}) < 2:
return None
return Verdict("variant_changes_output", "variants", sorted(labels),
f"the copies of {entry['name']} change what the core shows",
(record_id,))
shown = ", ".join(f"{run.label} {run.verdict}" for run in runs)
return Verdict("variant_changes_verdict", "variants", sorted(run.label for run in runs),
f"the copies of {entry['name']} do not boot alike: {shown}", ids(*runs))
def merge_evidence(existing: dict | None, verdict: Verdict) -> dict:
"""The entry's evidence: map with the verdict's record ids added under its field."""
def merge_evidence(existing: dict | None, verdict: Verdict,
keep: object = lambda rid: True) -> dict:
"""The entry's evidence: map with the verdict's record ids added under its
field. An id `keep` refuses is dropped from that field: a record from
before the oracle no longer proves anything."""
out = {key: list(value) for key, value in (existing or {}).items()}
ids = out.setdefault(verdict.field, [])
ids_ = [rid for rid in out.get(verdict.field, []) if keep(rid)]
for rid in verdict.evidence:
if rid not in ids:
ids.append(rid)
return {key: out[key] for key in sorted(out)}
if rid not in ids_:
ids_.append(rid)
out[verdict.field] = ids_
return {key: out[key] for key in sorted(out) if out[key]}
scripts/core_probe.py
+267 -16
View File
@@ -375,15 +375,25 @@ class Frontend:
self.unanswered: dict[int, int] = {}
self.input_queries = 0
self.input_pressed = 0
# What a run showed so far, one JSON line per event, flushed at once:
# a core that kills the worker leaves its logs and frames behind.
events = spec.get("events")
self.events = open(events, "a", encoding="utf-8", buffering=1) if events else None # noqa: SIM115
self.callbacks = (
EnvFn(self.environment), VideoFn(self.video_refresh), AudioFn(self.audio_sample),
AudioBatchFn(self.audio_batch), PollFn(lambda: None), InputFn(self.input_state),
)
self.log_callback = RetroLogCallback(LogFn(self.log))
def event(self, kind: str, **values: object) -> None:
if self.events:
self.events.write(json.dumps({"event": kind, **values}) + "\n")
def log(self, level: int, fmt: bytes, *args: int) -> None:
name = LOG_LEVELS[level] if 0 <= level < len(LOG_LEVELS) else str(level)
self.logs.append({"frame": self.frame, "level": name, "text": render_printf(fmt or b"", args)})
line = {"frame": self.frame, "level": name, "text": render_printf(fmt or b"", args)}
self.logs.append(line)
self.event("log", **line)
def environment(self, cmd: int, data: int) -> bool:
cmd &= ~ENV_EXPERIMENTAL
@@ -446,8 +456,10 @@ class Frontend:
if every and self.frame % every == 0:
digest = hashlib.sha1(self.last_frame).hexdigest()
self.frame_hashes.append([self.frame, digest])
if digest not in self.uniform and uniform_frame(self.last_frame, PIXEL_BYTES[self.pixel_format]):
flat = digest in self.uniform or uniform_frame(self.last_frame, PIXEL_BYTES[self.pixel_format])
if flat:
self.uniform.add(digest)
self.event("frame", frame=self.frame, sha1=digest, uniform=flat)
png_dir = self.spec.get("png_dir")
if png_dir and self.last_frame:
rgb = to_rgb(self.pixel_format, *self.last_size, self.last_frame)
@@ -483,6 +495,7 @@ def run_worker(spec: dict) -> dict:
getattr(core, name)(fn)
info = RetroSystemInfo()
core.retro_get_system_info(ctypes.byref(info))
front.event("info", library_name=info.library_name.decode(), library_version=info.library_version.decode())
core.retro_init()
game = None
if spec.get("content"):
@@ -494,7 +507,9 @@ def run_worker(spec: dict) -> dict:
front.option_buffers.append(holder)
game.data, game.size = ctypes.cast(holder, ctypes.c_void_p), len(blob)
core.retro_load_game.restype = ctypes.c_bool
if not core.retro_load_game(ctypes.byref(game) if game else None):
loaded = bool(core.retro_load_game(ctypes.byref(game) if game else None))
front.event("loaded", value=loaded)
if not loaded:
return {"loaded": False, "logs": front.logs, "library_name": info.library_name.decode()}
# A frontend plugs a joypad into each port after loading, as RetroArch does.
for port in range(spec["joypad_ports"]):
@@ -619,10 +634,41 @@ def core_identity(args: argparse.Namespace) -> dict:
# Runs (parent side)
# --------------------------------------------------------------------------
@dataclass(frozen=True)
class Synthetic:
"""A wrong file made for a negative test: `@zeros:SIZE` is SIZE zero
bytes, `@head:SIZE:PATH` the first SIZE bytes of PATH."""
kind: str
size: int
source: Path | None = None
@classmethod
def parse(cls, text: str) -> Synthetic:
kind, _, rest = text[1:].partition(":")
size_text, _, source = rest.partition(":")
if kind not in ("zeros", "head") or not size_text.isdigit() or (kind == "head") != bool(source):
raise argparse.ArgumentTypeError(
f"{text!r}: a made file is @zeros:SIZE or @head:SIZE:PATH")
return cls(kind, int(size_text), Path(source) if source else None)
def label(self, dest: str) -> str:
if self.kind == "zeros":
return f"{dest}-zeros-{self.size}"
return f"{dest}-head-{self.size}-of-{self.source.name}"
def write(self, out: Path) -> None:
if self.kind == "zeros":
out.write_bytes(bytes(self.size))
return
with self.source.open("rb") as handle:
out.write_bytes(handle.read(self.size))
@dataclass
class Variant:
label: str
files: dict[str, Path | None] = field(default_factory=dict)
files: dict[str, Path | Synthetic | None] = field(default_factory=dict)
def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> dict[str, str | None]:
@@ -632,10 +678,14 @@ def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> d
for dest, src in {**base, **variant.files}.items():
out = target / dest
if src is None:
out.unlink(missing_ok=True)
staged[dest] = None
continue
out.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(src, out)
if isinstance(src, Synthetic):
src.write(out)
else:
shutil.copyfile(src, out)
staged[dest] = file_digest(out)
return staged
@@ -729,6 +779,45 @@ def run_group(command: list[str], timeout: float, env: dict[str, str]) -> subpro
return subprocess.CompletedProcess(command, proc.returncode, out, err)
WORKER_TRACEBACK = "Traceback (most recent call last)"
def exit_description(returncode: int) -> dict:
"""The status of a worker that died. A signal reaches the probe as a
negative code, or as 128 + its number through bubblewrap's init."""
number = -returncode if returncode < 0 else returncode - 128 if 128 < returncode < 160 else 0
name = None
if number:
with contextlib.suppress(ValueError):
name = signal.Signals(number).name
return {"status": returncode, "signal": name}
def partial_result(events: Path, spec: dict) -> dict:
"""A repetition rebuilt from the events a worker wrote before it died."""
lines = []
if events.is_file():
for line in events.read_text(encoding="utf-8", errors="replace").splitlines():
with contextlib.suppress(ValueError):
lines.append(json.loads(line))
info = next((e for e in lines if e.get("event") == "info"), {})
loaded = next((e["value"] for e in lines if e.get("event") == "loaded"), False)
frames = [e for e in lines if e.get("event") == "frame"]
result: dict = {
"loaded": bool(loaded),
"library_name": info.get("library_name", ""),
"logs": [{k: e[k] for k in ("frame", "level", "text")} for e in lines if e.get("event") == "log"],
}
if info.get("library_version"):
result["library_version"] = info["library_version"]
if spec["frame_hash_every"]:
result["observables"] = {
"frame_hashes": [[e["frame"], e["sha1"]] for e in frames],
"uniform_frames": sorted({e["sha1"] for e in frames if e.get("uniform")}),
}
return result
def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None,
png_dir: Path | None) -> dict:
save_dir = scratch / "save"
@@ -748,6 +837,7 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou
"png_dir": str(png_dir) if png_dir else None,
"state_out": str(state_out) if state_out else None,
"env": dict(args.env),
"events": str((scratch / "events.jsonl").resolve()),
}
spec_path, out_path, trace_path = scratch / "spec.json", scratch / "result.json", scratch / "strace.txt"
spec_path.write_text(json.dumps(spec), encoding="utf-8")
@@ -764,10 +854,20 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou
if args.require_sandbox and isolation != SANDBOX:
raise ProbeError(f"refused to run the core outside a sandbox: {isolation}")
# The core sees the variables it was given and nothing of the host's.
proc = run_group(prefix + command, args.timeout, dict(args.env))
if proc.returncode != 0 or not out_path.exists():
raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}")
result = json.loads(out_path.read_text(encoding="utf-8"))
try:
proc = run_group(prefix + command, args.timeout, dict(args.env))
except subprocess.TimeoutExpired:
result = partial_result(scratch / "events.jsonl", spec)
result["exit"] = {"status": None, "signal": "SIGKILL", "timeout_s": args.timeout}
else:
if proc.returncode == 0 and out_path.exists():
result = json.loads(out_path.read_text(encoding="utf-8"))
elif WORKER_TRACEBACK in proc.stderr:
# The probe's own code failed: no verdict on the core is drawn.
raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}")
else:
result = partial_result(scratch / "events.jsonl", spec)
result["exit"] = exit_description(proc.returncode)
places = [(str(system_dir.resolve()), "<system>"), (str(save_dir.resolve()), "<save>")]
if args.content:
places.append((str(Path(args.content).resolve().parent), "<content>"))
@@ -820,6 +920,10 @@ def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Varian
if state_out:
state_out.parent.mkdir(parents=True, exist_ok=True)
rep = run_once(args, system_dir, scratch, state_out, png_dir)
oracle = oracle_for(args.oracle_data, args.content_sha1, rep.get("library_name"))
rep["verdict"], rep["verdict_evidence"] = judge(rep, oracle)
if rep.get("exit"):
flags.append(f"repetition {index}: {rep['verdict_evidence']['detail']}")
if not rep.get("loaded"):
flags.append(f"repetition {index}: retro_load_game returned false")
for entry in rep.get("files_opened") or []:
@@ -833,11 +937,14 @@ def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Varian
if rep.get("observables") != reference:
differing = sorted(k for k in (reference or {}) if (rep.get("observables") or {}).get(k) != reference[k])
flags.append(f"repetition {index} disagrees with repetition 0 on {', '.join(differing)}")
verdict, evidence = variant_verdict(reps)
return {
"label": variant.label,
"system_files": {dest: sha for dest, sha in sorted(staged.items())},
"repetitions": reps,
"flags": flags,
"verdict": verdict,
"verdict_evidence": evidence,
}
@@ -894,12 +1001,18 @@ def build_parser() -> argparse.ArgumentParser:
help="refuse to record when the 1-minute load average exceeds this "
"(default: a quarter of the CPU count)")
common.add_argument("--work-dir", default=str(DEFAULT_WORK_DIR), help="scratch directory")
common.add_argument("--timeout", type=int, default=1800, help="seconds per repetition")
common.add_argument("--timeout", type=int, default=1800,
help="seconds per repetition, after which the run is killed and judged a hang")
common.add_argument("--oracle", default=None, metavar="FILE",
help="reference frames and error patterns the verdicts are judged against "
"(default: tests/campaign/oracle.yml)")
common.add_argument("--no-trace", action="store_true", help="do not run the worker under strace")
common.add_argument("--require-sandbox", action="store_true",
help="refuse to run the core unless bubblewrap can isolate it")
sub.add_parser("run", parents=[common], help="one execution")
run = sub.add_parser("run", parents=[common], help="one execution")
run.add_argument("--repeat", type=int, default=1,
help="runs, each in a fresh process: the verdict is the one they all agree on")
bench = sub.add_parser("bench", parents=[common], help="timed repetitions")
bench.add_argument("--repeat", type=int, default=MIN_BENCH_REPETITIONS,
help=f"timed repetitions, each in a fresh process (at least {MIN_BENCH_REPETITIONS} to record)")
@@ -941,12 +1054,16 @@ def validate_args(args: argparse.Namespace) -> list[str]:
for dest, src in args.system_file:
if not Path(src).is_file():
problems.append(f"--system-file {dest}={src}: not a file")
if args.repeat < 1:
problems.append("--repeat must be >= 1")
if args.frames < 1 or args.warmup < 0:
problems.append("--frames must be >= 1 and --warmup >= 0")
if args.require_sandbox:
isolation = sandbox_argv([])[1]
if isolation != SANDBOX:
problems.append(f"--require-sandbox: {isolation}")
if args.oracle and not Path(args.oracle).is_file():
problems.append(f"--oracle {args.oracle}: not a file")
if args.png_dir and not args.frame_hash_every:
problems.append("--png-dir needs --frame-hash-every")
if args.record:
@@ -970,7 +1087,15 @@ def validate_args(args: argparse.Namespace) -> list[str]:
if len(dests) != 1:
problems.append("--vary must name the same DEST in every occurrence")
for dest, src in args.vary:
if src and not Path(src).is_file():
if src.startswith("@"):
try:
made = Synthetic.parse(src)
except argparse.ArgumentTypeError as exc:
problems.append(f"--vary {dest}: {exc}")
continue
if made.source and not made.source.is_file():
problems.append(f"--vary {dest}={src}: {made.source} is not a file")
elif src and not Path(src).is_file():
problems.append(f"--vary {dest}={src}: not a file")
names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var}
if args.frame_hash_every:
@@ -986,11 +1111,12 @@ def variants_for(args: argparse.Namespace) -> list[Variant]:
out: list[Variant] = []
seen: set[str] = set()
for dest, src in args.vary:
label = Path(src).name if src else f"{dest}-absent"
made = Synthetic.parse(src) if src.startswith("@") else None
label = made.label(dest) if made else Path(src).name if src else f"{dest}-absent"
while label in seen:
label += "+"
seen.add(label)
out.append(Variant(label, {dest: Path(src) if src else None}))
out.append(Variant(label, {dest: made or (Path(src) if src else None)}))
return out
@@ -1036,6 +1162,125 @@ def check_result(variants: list[dict], observable: str) -> dict:
}
# --------------------------------------------------------------------------
# Boot oracle: what a run reached, from positive evidence
# --------------------------------------------------------------------------
BOOTS, BIOS_ERROR, CRASH, HANG, UNKNOWN = "boots", "bios_error", "crash", "hang", "unknown"
VERDICTS = (BOOTS, BIOS_ERROR, CRASH, HANG, UNKNOWN)
DEFAULT_ORACLE = ROOT / "tests" / "campaign" / "oracle.yml"
def oracle_for(data: dict, content_sha1: str | None, library_name: str | None) -> dict:
"""The oracle entries that judge one content on one core: its reference
frames, the error patterns every core shares and the core's own, and the
core's known error screens."""
core = (data.get("cores") or {}).get(library_name or "") or {}
references: list[dict] = []
for item in data.get("content") or ():
if content_sha1 and item.get("sha1") == content_sha1:
block = (item.get("references") or {}).get(library_name or "") or {}
references = [{"sha1": f["sha1"], "stage": f["stage"], "shows": f["shows"]}
for f in block.get("frames") or ()]
patterns = [p["pattern"] for p in (data.get("error_patterns") or ())]
patterns += [p["pattern"] for p in core.get("error_patterns") or ()]
screens = [{"sha1": s["sha1"], "shows": s["shows"]} for s in core.get("error_screens") or ()]
return {"references": references, "error_patterns": patterns, "error_screens": screens}
def sampled(rep: dict) -> list[tuple[int, str]]:
observables = rep.get("observables") or {}
return [(int(frame), digest) for frame, digest in observables.get("frame_hashes") or ()]
def judge(rep: dict, oracle: dict) -> tuple[str, dict]:
"""The verdict of one repetition and the evidence that decided it.
In order: a worker killed after the timeout is a hang and one that died
a crash, from the process status. A reference frame reached is a boot.
Without one, an error pattern of the core log or a known error screen
is a BIOS error. Every sampled frame of the second half one colour is a
hang. Anything else is unknown: a run is never called a boot because
its frames differ from another run's.
"""
ended = rep.get("exit")
if ended and ended.get("timeout_s") is not None:
return HANG, {"rule": "timeout", "detail": f"killed after {ended['timeout_s']} s",
"exit_status": ended.get("status"), "signal": ended.get("signal")}
if ended:
how = f"signal {ended['signal']}" if ended.get("signal") else f"exit status {ended.get('status')}"
return CRASH, {"rule": "exit_status", "detail": f"the worker died: {how}",
"exit_status": ended.get("status"), "signal": ended.get("signal")}
frames = sampled(rep)
final = (rep.get("observables") or {}).get("frame_sha1")
if final:
frames.append((int(rep.get("frames", 0)) + int(rep.get("warmup", 0)), final))
references = {r["sha1"]: r for r in oracle.get("references") or ()}
reached = [(frame, digest) for frame, digest in frames if digest in references]
if reached:
frame, digest = reached[0]
stages = list(dict.fromkeys(references[d]["stage"] for _, d in reached))
return BOOTS, {"rule": "reference_frame",
"detail": f"frame {frame} shows {references[digest]['shows']}",
"frame": frame, "sha1": digest, "stages": stages}
for pattern in oracle.get("error_patterns") or ():
for line in rep.get("logs") or ():
if re.search(pattern, line["text"]):
return BIOS_ERROR, {"rule": "error_pattern",
"detail": f"the core logged an error at frame {line['frame']}",
"pattern": pattern, "log": line["text"], "frame": line["frame"]}
screens = {s["sha1"]: s for s in oracle.get("error_screens") or ()}
for frame, digest in frames:
if digest in screens:
return BIOS_ERROR, {"rule": "error_screen",
"detail": f"frame {frame} shows {screens[digest]['shows']}",
"frame": frame, "sha1": digest}
if not rep.get("loaded"):
return UNKNOWN, {"rule": "no_evidence",
"detail": "retro_load_game returned false and no error pattern matched"}
uniform = set((rep.get("observables") or {}).get("uniform_frames") or ())
tail = [digest for frame, digest in sampled(rep)][len(sampled(rep)) // 2:]
if tail and all(digest in uniform for digest in tail):
return HANG, {"rule": "uniform_tail",
"detail": f"the last {len(tail)} sampled frames are one colour each"}
if not oracle.get("references"):
return UNKNOWN, {"rule": "no_evidence", "detail": "no reference frame is known for this "
"content on this core"}
return UNKNOWN, {"rule": "no_evidence", "detail": "no reference frame reached, no error pattern "
"and no error screen"}
def variant_verdict(reps: Sequence[dict]) -> tuple[str, dict]:
"""The verdict all repetitions agree on, else unknown."""
verdicts = [rep["verdict"] for rep in reps]
if len(set(verdicts)) == 1:
first = reps[0]["verdict_evidence"]
evidence = {"rule": "repetitions_agree",
"detail": f"{len(reps)} of {len(reps)} repetitions: {first['detail']}",
"repetitions": verdicts}
for key in ("frame", "sha1", "stages", "pattern", "log", "exit_status", "signal"):
if key in first:
evidence[key] = first[key]
return verdicts[0], evidence
return UNKNOWN, {"rule": "repetitions_disagree",
"detail": f"the repetitions disagree: {', '.join(verdicts)}",
"repetitions": verdicts}
def load_oracle(path: Path | None) -> tuple[dict, dict | None]:
"""The oracle file's data and its identity, or nothing without a file."""
if path is None or not path.is_file():
return {}, None
import yaml # noqa: PLC0415
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
try:
shown = path.resolve().relative_to(ROOT).as_posix()
except ValueError:
shown = path.name
return data, {"file": shown, "sha256": file_digest(path, "sha256")}
def default_emulator(core_file: Path, library_name: str | None) -> str:
"""The profile whose cores: names the .so, else the library name as a slug."""
stem = core_file.name.split(".")[0].removesuffix("_libretro")
@@ -1097,8 +1342,10 @@ def probe(args: argparse.Namespace) -> int:
print(f"ERROR: load average {load_before[0]} exceeds {max_load}: the machine is busy, "
f"nothing recorded (raise --max-load to override)", file=sys.stderr)
return 3
repetitions = 1 if args.mode == "run" else args.repeat
repetitions = args.repeat
base = {dest: Path(src) for dest, src in args.system_file}
args.oracle_data, oracle_identity = load_oracle(Path(args.oracle) if args.oracle else DEFAULT_ORACLE)
args.content_sha1 = file_digest(Path(args.content)) if args.content else None
work = Path(tempfile.mkdtemp(prefix=f"{args.mode}-", dir=_ensure(Path(args.work_dir))))
try:
variants = [run_variant(args, base, variant, work / f"v{i}", repetitions)
@@ -1136,6 +1383,9 @@ def probe(args: argparse.Namespace) -> int:
+ (["frame_hashes", "uniform_frames"] if args.frame_hash_every else []),
"frame_hash_every": args.frame_hash_every,
"joypad_ports": args.joypad_ports,
"oracle": ({**oracle_identity, **oracle_for(args.oracle_data, args.content_sha1,
first.get("library_name"))}
if oracle_identity else None),
},
"variants": variants,
"flags": [f"{v['label']}: {flag}" for v in variants for flag in v["flags"]],
@@ -1150,7 +1400,8 @@ def probe(args: argparse.Namespace) -> int:
if args.mode == "check":
record["check"] = {"claim": args.claim, "method": args.method, "observable": args.observable,
"variable": args.vary[0][0],
"result": check_result(variants, args.observable)}
"result": {**check_result(variants, args.observable),
"verdicts": {v["label"]: v["verdict"] for v in variants}}}
try:
record["emulator"] = args.emulator or default_emulator(Path(args.core), first.get("library_name"))
except ProbeError as exc:
scripts/measurement_records.py
+1 -1
View File
@@ -27,7 +27,7 @@ from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
RECORD_VERSION = 3
RECORD_VERSION = 4
# 3: before the boot oracle. 4: every repetition and variant carries a verdict.
RECORD_VERSIONS = (3, 4)
RECORDS_DIR = "measurements"