diff --git a/automation/jobs.yml b/automation/jobs.yml index 4d5de8963..f78cb7eca 100644 --- a/automation/jobs.yml +++ b/automation/jobs.yml @@ -85,6 +85,32 @@ jobs: stdout: tmp/autorun/bios-integrity.json report: tmp/autorun/bios-integrity.json + campaign: + schedule: "*-*-* 06:00" + network: true + max_load_wait: 7200 + timeout: 12600 + steps: + - run: [python, scripts/campaign.py, run, --budget, 150m] + ok_codes: [0, 75] + stdout: tmp/autorun/campaign.json + gate: + - [python, scripts/campaign.py, gate] + commits: + - subject: "chore: add campaign measurements" + body: >- + Core probe records, findings and the pass summary written by + campaign.py, behind the measurement tests and the schemas. + paths: [measurements/*.jsonl] + - subject: "chore: apply fields the campaign proved" + body: >- + Fields the campaign's rule table settled from real runs, each with + the record ids in evidence:, behind verify and an emulator pack of + every changed profile. + paths: [emulators/*.yml] + report: tmp/autorun/campaign.json + scratch: [tmp/autorun, tmp/core_probe] + assets: schedule: "*-*-* 04:30" network: true diff --git a/schemas/campaign-finding.schema.json b/schemas/campaign-finding.schema.json new file mode 100644 index 000000000..44e0aa424 --- /dev/null +++ b/schemas/campaign-finding.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/campaign-finding.schema.json", + "title": "RetroBIOS campaign finding", + "description": "One line of measurements/findings.jsonl: what a real-condition run showed about one field of one profile entry, the rule that settled it or the reason it waits for review, and the records that show it. Lines are appended, never rewritten: the latest line of a key is its state.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "id", "found_at", "key", "emulator", "system", "file", "field", + "status", "rule", "profile_says", "run_shows", "statement", "evidence", "core", "content"], + "properties": { + "schema_version": { "const": 1 }, + "id": { + "type": "string", + "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$", + "description": "Date and the first 12 hex digits of the sha256 of the line without its id, in canonical JSON." + }, + "found_at": { "type": "string", "format": "date-time" }, + "key": { + "type": "string", + "minLength": 1, + "description": "emulator/system/file/field: the lines of one key are the history of one claim." + }, + "emulator": { "type": "string", "pattern": "^[a-z0-9]+(?:[-_][a-z0-9]+)*$" }, + "system": { "type": "string", "minLength": 1 }, + "file": { "type": "string", "minLength": 1 }, + "field": { "type": "string", "minLength": 1 }, + "status": { + "enum": ["applied", "open", "flagged", "resolved"], + "description": "applied: the rule table settled the field and the profile was changed. open: no rule settles it, it waits for review. flagged: a declared file the runs never opened. resolved: a later run found the profile in agreement." + }, + "rule": { "type": "string", "minLength": 1 }, + "profile_says": {}, + "run_shows": {}, + "statement": { "type": "string", "minLength": 1 }, + "evidence": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$" }, + "description": "Ids of the records in measurements/.jsonl that show it." + }, + "core": { + "type": "object", + "additionalProperties": false, + "required": ["file", "sha256", "library_version"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "library_version": { "type": ["string", "null"] } + } + }, + "content": { "type": ["string", "null"] } + } +} diff --git a/schemas/campaign-run.schema.json b/schemas/campaign-run.schema.json new file mode 100644 index 000000000..325cf67b1 --- /dev/null +++ b/schemas/campaign-run.schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/campaign-run.schema.json", + "title": "RetroBIOS campaign run", + "description": "One line of measurements/campaign-runs.jsonl: one pass of scripts/campaign.py, what it left done, failed and pending, and the records and findings it added. Lines are appended, never rewritten.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "id", "started_at", "ended_at", "result", "detail", "counts", + "pending_reasons", "failed", "records", "findings"], + "properties": { + "schema_version": { "const": 1 }, + "id": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$" }, + "started_at": { "type": "string", "format": "date-time" }, + "ended_at": { "type": "string", "format": "date-time" }, + "result": { + "enum": ["complete", "budget", "paused", "stopped"], + "description": "complete: nothing runnable was left. budget: the time budget ended the pass. paused: a busy host or a full disk ended it, the next pass resumes. stopped: interrupted." + }, + "detail": { "type": "string" }, + "counts": { + "type": "object", + "additionalProperties": false, + "required": ["done", "failed", "pending"], + "properties": { + "done": { "type": "integer", "minimum": 0 }, + "failed": { "type": "integer", "minimum": 0 }, + "pending": { "type": "integer", "minimum": 0 } + } + }, + "pending_reasons": { + "type": "object", + "additionalProperties": { "type": "integer", "minimum": 1 }, + "description": "Pending checks counted by reason." + }, + "failed": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["emulator", "check", "reason"], + "properties": { + "emulator": { "type": "string" }, + "check": { "type": "string" }, + "reason": { "type": "string" } + } + } + }, + "records": { + "type": "array", + "items": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$" } + }, + "findings": { + "type": "object", + "additionalProperties": { "type": "integer", "minimum": 0 }, + "description": "Finding lines added by this pass, counted by status." + } + } +} diff --git a/schemas/core-probe-record.schema.json b/schemas/core-probe-record.schema.json index 8b5fb74f4..00594716d 100644 --- a/schemas/core-probe-record.schema.json +++ b/schemas/core-probe-record.schema.json @@ -125,6 +125,11 @@ "string", "null" ] + }, + "artifact": { + "type": "string", + "format": "uri", + "description": "Where the .so was downloaded from, such as a libretro buildbot nightly. Absent for a core built locally." } } }, diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 3b6062aa6..cada7d28d 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -478,6 +478,18 @@ "hle_fallback": { "type": "boolean" }, + "evidence": { + "description": "Real-condition runs that settled a field of this entry, by field: the ids of the core probe records in measurements/.jsonl. Written by scripts/campaign.py when its rule table settles the field.", + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "required": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$"}}, + "hle_fallback": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$"}}, + "path": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$"}}, + "aliases": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$"}} + } + }, "adler32_byteswap": { "type": "boolean" }, diff --git a/scripts/autorun.py b/scripts/autorun.py index 9632f4f61..693bba54e 100644 --- a/scripts/autorun.py +++ b/scripts/autorun.py @@ -54,6 +54,7 @@ JOBS_FILE = REPO_ROOT / "automation" / "jobs.yml" AUTORUN_DIR = Path(".cache") / "autorun" DEFAULT_LOCK = Path.home() / ".local" / "state" / "retrobios" / "auto.lock" LOCK_ENV = "RETROBIOS_AUTO_LOCK" +LOCK_HELD_ENV = "RETROBIOS_AUTO_LOCK_HELD" BRANCH = "autorun" REMOTE = "origin" @@ -726,6 +727,9 @@ def set_alert(state: dict, name: str, active: bool, detail: str, now: str) -> No def job_env(job: Job) -> dict[str, str]: env = os.environ.copy() env["RETROBIOS_AUTO"] = "1" + if job.lock: + # A step that takes the host lock itself (campaign.py) runs under ours. + env[LOCK_HELD_ENV] = "1" if job.network: sys.path.insert(0, str(REPO_ROOT / "scripts")) from common import github_token diff --git a/scripts/campaign.py b/scripts/campaign.py new file mode 100644 index 000000000..ebdff09a3 --- /dev/null +++ b/scripts/campaign.py @@ -0,0 +1,1511 @@ +#!/usr/bin/env python3 +"""Run the real-condition campaign unattended: cores, content, checks, rules. + +For each emulator profile in scope, the campaign takes the core (the libretro +x86_64 Linux nightly, else a build of the profile's pinned commit), the test +content of each of its systems (tests/campaign/content.yml, fetched through +romfetch when it is not on disk), and runs core_probe.py: + + files every declared file the collection holds, staged; which ones + the core opens + boot the content with no file against the content with the declared + file, compared on the frames the core shows + path a declared file staged where the core looked for it + alias a declared file staged under a name the core looked for + variants each .variants/ copy of a file the core opened + bench the content timed, warm-up excluded, 5 repetitions + +Every run is a record in measurements/.jsonl. The fixed rule table +(campaign_rules.py) turns records into verdicts. A verdict the table settles +is written into the profile with the record ids as `evidence:`. Any other +goes to measurements/findings.jsonl, where the site lists it as open. Each +pass adds one line to measurements/campaign-runs.jsonl. + +State is .cache/campaign/state.json of the main checkout: per check done, +failed or pending with the reason, and the core sha256 and profile digest it +ran against. A pass skips what is done unless the core build or the profile +changed. The campaign holds the host-wide automation lock, waits for an idle +host, stops cleanly on a busy host or a full disk, and resumes on the next +pass. Under autorun (`campaign` job) the lock is the job's. + +Usage: + python scripts/campaign.py run [--only CORE,...] [--system ID] [--budget 6h] + python scripts/campaign.py run --dry-run + python scripts/campaign.py run --commit [--commit-every N] + python scripts/campaign.py status [--json] + python scripts/campaign.py findings [--all] [--json] + python scripts/campaign.py gate + python scripts/campaign.py install-timer [--enable] +""" + +from __future__ import annotations + +import argparse +import contextlib +import datetime as dt +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request +import zipfile +from collections.abc import Callable, Iterator, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import autorun +import campaign_profiles +import campaign_rules as rules +import measurement_records as mr + +ROOT = Path(__file__).resolve().parent.parent +CONTENT_FILE = ROOT / "tests" / "campaign" / "content.yml" +BUILDBOT = "https://buildbot.libretro.com/nightly/linux/x86_64/latest/" +LOCK_HELD_ENV = autorun.LOCK_HELD_ENV +ROMFETCH_ENV = "RETROBIOS_ROMFETCH" +STATE_VERSION = 1 + +DONE = "done" +FAILED = "failed" +PENDING = "pending" + +CRASH_ATTEMPTS = 3 +NETWORK_DELAYS = (10, 60, 300) +MAX_ALIAS_CANDIDATES = 6 +BENCH_WARMUP = 600 +BENCH_FRAMES = 1800 +BENCH_REPEAT = 5 +CHECK_REPEAT = 2 +PROBE_TIMEOUT = 900 +BUILD_TIMEOUT = 3600 +LOG_DAYS = 30 +ROMFETCH_TRANSIENT = 3 + + +class Pause(RuntimeError): + """The host cannot run measurements now: the pass stops and resumes later.""" + + +class Unavailable(RuntimeError): + """A core or a piece of content the campaign cannot get.""" + + +# -------------------------------------------------------------------------- +# Small helpers +# -------------------------------------------------------------------------- + +def now_iso() -> str: + return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat() + + +def slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") or "x" + + +def parse_budget(text: str) -> float: + match = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([smh]?)\s*", text or "") + if not match: + raise argparse.ArgumentTypeError(f"--budget {text!r}: a number with s, m or h") + return float(match[1]) * {"": 1, "s": 1, "m": 60, "h": 3600}[match[2]] + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def file_sha1(path: Path) -> str: + digest = hashlib.sha1() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def profile_digest(profile: dict) -> str: + """What a check depends on in a profile: its entries, evidence left out, + so the campaign's own evidence never makes it run again.""" + entries = [{k: v for k, v in entry.items() if k != "evidence"} + for entry in profile.get("files") or []] + body = {"files": entries, "cores": profile.get("cores"), "type": profile.get("type"), + "systems": profile.get("systems")} + return hashlib.sha1(mr.canonical(body)).hexdigest() + + +def entry_systems(entry: dict, profile: dict) -> list[str]: + if entry.get("systems"): + return list(entry["systems"]) + if entry.get("system"): + return [entry["system"]] + return list(profile.get("systems") or []) + + +def entry_dest(entry: dict) -> str: + """Where the core reads the entry, relative to its system directory.""" + path = entry.get("path") + if isinstance(path, str) and path and not path.endswith("/"): + return path + return entry["name"] + + +def region_matches(entry: dict, region: str | None) -> bool: + regions = entry.get("region") + if not regions or not region: + return True + if isinstance(regions, str): + regions = [regions] + return region in regions or "world" in regions + + +def is_bios(entry: dict) -> bool: + return entry.get("category", "bios") == "bios" and entry.get("type") != "directory" + + +# -------------------------------------------------------------------------- +# Host +# -------------------------------------------------------------------------- + +@dataclass +class Host: + """What the campaign reads from the machine and the network, replaced in tests.""" + + loadavg: Callable[[], tuple[float, float, float]] = os.getloadavg + sleep: Callable[[float], None] = time.sleep + clock: Callable[[], float] = time.monotonic + cpu_count: int = os.cpu_count() or 1 + free_gb: Callable[[Path], float] = autorun.free_gb + poll: float = 60.0 + run: Callable[..., subprocess.CompletedProcess] = subprocess.run + urlopen: Callable[..., object] = urllib.request.urlopen + + @property + def max_load(self) -> float: + return self.cpu_count / 4 + + +@dataclass +class Settings: + budget: float = 6 * 3600 + max_load_wait: float = 3600 + min_free_gb: float = 30 + cache_cap_gb: float = 20 + probe_timeout: int = PROBE_TIMEOUT + check_repeat: int = CHECK_REPEAT + bench_repeat: int = BENCH_REPEAT + bench_warmup: int = BENCH_WARMUP + bench_frames: int = BENCH_FRAMES + romfetch: str = "" + network: bool = True + delays: tuple[float, ...] = NETWORK_DELAYS + echo: bool = True + + +@dataclass +class Paths: + """Where the campaign writes: records and profiles in the checkout it + runs from, state, cores, logs and content in the main checkout's .cache.""" + + root: Path + home: Path + content: Path + db: Path + content_file: Path = CONTENT_FILE + + @property + def state(self) -> Path: + return self.home / "state.json" + + @property + def logs(self) -> Path: + return self.home / "logs" + + @property + def emulators(self) -> Path: + return self.root / "emulators" + + @classmethod + def default(cls) -> Paths: + main = autorun.main_checkout() + return cls(root=ROOT, home=main / ".cache" / "campaign", + content=main / ".cache" / "test-content", db=ROOT / "database.json") + + +@contextlib.contextmanager +def host_lock(lock_path: Path) -> Iterator[None]: + """The automation lock, unless the autorun job running us holds it.""" + if os.environ.get(LOCK_HELD_ENV): + yield + return + with autorun.global_lock(lock_path): + yield + + +# -------------------------------------------------------------------------- +# State +# -------------------------------------------------------------------------- + +def load_state(path: Path) -> dict: + try: + state = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + state = {} + state.setdefault("version", STATE_VERSION) + for key in ("checks", "cores", "content", "builds", "emulators"): + state.setdefault(key, {}) + return state + + +def save_state(path: Path, state: dict) -> None: + autorun.save_state(path, state) + + +def should_run(previous: dict | None, core_sha: str, digest: str) -> bool: + """A check runs again when it never ran, is pending, or ran against + another core build or another version of the profile.""" + if not previous: + return True + if previous.get("status") == PENDING: + return True + return previous.get("core_sha256") != core_sha or previous.get("profile_digest") != digest + + +# -------------------------------------------------------------------------- +# Content: what each system is tested with, and where it comes from +# -------------------------------------------------------------------------- + +@dataclass(frozen=True) +class Content: + system: str + romfetch_system: str + name: str + catalog: str + reason: str + region: str | None + load: str + frames: int + frame_hash_every: int + pinned: bool + boot_file: str | None + + @property + def key(self) -> str: + return f"{self.romfetch_system}/{self.name}" + + +def load_content(path: Path) -> dict[str, list[Content]]: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + out: dict[str, list[Content]] = {} + for system, spec in (data.get("systems") or {}).items(): + for item in spec.get("content") or (): + out.setdefault(system, []).append(Content( + system=system, romfetch_system=str(spec["romfetch"]), name=str(item["name"]), + catalog=str(item["catalog"]), reason=str(item["reason"]).strip(), + region=item.get("region"), load=str(item.get("load") or ""), + frames=int(item.get("frames", 3600)), + frame_hash_every=int(item.get("frame_hash_every", 10)), + pinned=bool(item.get("pinned", False)), boot_file=spec.get("boot_file"), + )) + return out + + +def load_file(folder: Path, load: str) -> Path: + """The file handed to the core.""" + if load == "cue": + cues = sorted(folder.glob("*.cue")) + if len(cues) != 1: + raise Unavailable(f"{folder.name}: {len(cues)} .cue files, one expected") + return cues[0] + if load: + return folder / load + files = [p for p in sorted(folder.iterdir()) if p.is_file() and p.name != "identity.json"] + if len(files) != 1: + raise Unavailable(f"{folder.name}: {len(files)} files, name one with load:") + return files[0] + + +def verify_identity(folder: Path) -> dict: + """The identity.json romfetch wrote, each listed file hashed again.""" + try: + identity = json.loads((folder / "identity.json").read_text(encoding="utf-8")) + except FileNotFoundError as exc: + raise Unavailable(f"{folder}: no identity.json") from exc + if identity.get("status") != "verified": + raise Unavailable(f"{folder.name}: identity status {identity.get('status')!r}") + for item in identity.get("files") or (): + path = folder / item["name"] + if not path.is_file() or path.stat().st_size != item["size"] or file_sha1(path) != item["sha1"]: + raise Unavailable(f"{folder.name}: {item['name']} does not match its identity") + return identity + + +def folder_size(folder: Path) -> int: + return sum(p.stat().st_size for p in folder.rglob("*") if p.is_file()) + + +class ContentStore: + """Content on disk, its users counted, the least recently used evicted + past the cap. Only what the campaign fetched is ever deleted, and never + a pinned entry.""" + + def __init__(self, campaign: Campaign) -> None: + self.c = campaign + self.state: dict = campaign.state["content"] + self.refs: dict[str, set[str]] = {} + self.verified: set[str] = set() + + def folder(self, content: Content) -> Path: + return self.c.paths.content / content.romfetch_system / content.name + + def hold(self, content: Content, user: str) -> None: + self.refs.setdefault(content.key, set()).add(user) + + def get(self, content: Content) -> Path: + """The folder of the content, fetched when absent. Unavailable when not.""" + folder = self.folder(content) + entry = self.state.setdefault(content.key, {"fetched_by_campaign": False}) + entry["pinned"] = content.pinned + if not (folder / "identity.json").is_file(): + self.evict(keep={content.key}) + self.fetch(content) + entry["fetched_by_campaign"] = True + if content.key not in self.verified: + identity = verify_identity(folder) + entry["identity"] = {"catalog": identity.get("catalog"), "dat": identity.get("dat"), + "entry": identity.get("entry")} + self.verified.add(content.key) + entry["size"] = folder_size(folder) + entry["last_used"] = now_iso() + return folder + + def romfetch(self) -> str | None: + configured = self.c.settings.romfetch or os.environ.get(ROMFETCH_ENV, "") + if configured: + return configured if Path(configured).is_file() or shutil.which(configured) else None + return shutil.which("romfetch") + + def fetch(self, content: Content) -> None: + command = self.romfetch() + if not command: + raise Unavailable(f"content: romfetch not found, {content.catalog} entry {content.name!r}") + if not self.c.settings.network: + raise Unavailable(f"content: offline, {content.catalog} entry {content.name!r}") + argv = [command, "get", content.romfetch_system, content.name, "--json", "-q", + "--content-dir", str(self.c.paths.content)] + for attempt, delay in enumerate((*self.c.settings.delays, None)): + proc = self.c.host.run(argv, capture_output=True, text=True, check=False, + timeout=6 * 3600) + if proc.returncode == 0: + self.c.log(f"romfetch {content.name}: {proc.stdout.strip()[:300]}") + return + code = "unknown" + with contextlib.suppress(ValueError, AttributeError): + code = json.loads(proc.stdout)["error"]["code"] + if proc.returncode != ROMFETCH_TRANSIENT or delay is None: + raise Unavailable(f"content: romfetch exit {proc.returncode} ({code}) " + f"for {content.name!r}") + self.c.log(f"romfetch {content.name}: exit {proc.returncode} ({code}), " + f"retry {attempt + 1} in {delay}s") + self.c.host.sleep(delay) + + def release(self, content: Content, user: str) -> None: + users = self.refs.get(content.key, set()) + users.discard(user) + if not users: + self.drop(content.key) + + def deletable(self, key: str) -> bool: + entry = self.state.get(key) or {} + return bool(entry.get("fetched_by_campaign")) and not entry.get("pinned") + + def drop(self, key: str) -> bool: + if not self.deletable(key) or self.refs.get(key): + return False + folder = self.c.paths.content / key + if folder.is_dir(): + shutil.rmtree(folder) + self.c.log(f"content {key}: deleted, no check left needs it") + self.state.pop(key, None) + self.verified.discard(key) + return True + + def evict(self, keep: set[str]) -> list[str]: + """Least recently used first, until the campaign's content fits the cap.""" + cap = self.c.settings.cache_cap_gb * 1024 ** 3 + held = sorted((entry.get("last_used", ""), key) for key, entry in self.state.items() + if self.deletable(key) and key not in keep) + total = sum(entry.get("size", 0) for key, entry in self.state.items() if self.deletable(key)) + evicted = [] + for _, key in held: + if total <= cap: + break + size = self.state[key].get("size", 0) + folder = self.c.paths.content / key + if folder.is_dir(): + shutil.rmtree(folder) + self.state.pop(key, None) + self.verified.discard(key) + total -= size + evicted.append(key) + self.c.log(f"content {key}: evicted, the cache is over {self.c.settings.cache_cap_gb} GB") + return evicted + + +# -------------------------------------------------------------------------- +# Cores +# -------------------------------------------------------------------------- + +@dataclass(frozen=True) +class Core: + name: str + path: Path + sha256: str + artifact: str | None = None + repo: str | None = None + commit: str | None = None + build_command: str | None = None + + +def safe_extract_so(archive: Path, member: str, target: Path) -> None: + with zipfile.ZipFile(archive) as handle: + names = handle.namelist() + if member not in names: + raise Unavailable(f"{archive.name}: no {member} inside ({', '.join(names[:5])})") + target.parent.mkdir(parents=True, exist_ok=True) + scratch = target.with_suffix(".part") + with handle.open(member) as src, scratch.open("wb") as dst: + shutil.copyfileobj(src, dst) + os.replace(scratch, target) + + +class Cores: + """Buildbot nightlies, downloaded once per build, else local builds of + the profile's pinned commit.""" + + def __init__(self, campaign: Campaign) -> None: + self.c = campaign + self.listing: set[str] | None = None + + def retrying(self, what: str, call: Callable[[], object]) -> object: + for attempt, delay in enumerate((*self.c.settings.delays, None)): + try: + return call() + except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as exc: + if isinstance(exc, urllib.error.HTTPError) and exc.code < 500 and exc.code != 429: + raise Unavailable(f"{what}: HTTP {exc.code}") from exc + if delay is None: + raise Unavailable(f"{what}: {exc}") from exc + self.c.log(f"{what}: {exc}, retry {attempt + 1} in {delay}s") + self.c.host.sleep(delay) + raise AssertionError("unreachable") + + def buildbot(self) -> set[str]: + if self.listing is None: + if not self.c.settings.network: + self.listing = set() + else: + def read() -> bytes: + with self.c.host.urlopen(BUILDBOT, timeout=120) as response: + return response.read() + page = self.retrying("buildbot listing", read).decode("utf-8", "replace") + self.listing = set(re.findall(r"([A-Za-z0-9_]+)_libretro\.so\.zip", page)) + return self.listing + + def names(self, profile: dict) -> list[str]: + listing = self.buildbot() + return sorted({str(c) for c in profile.get("cores") or () if str(c) in listing}) + + def nightly(self, name: str, profile: dict) -> Core: + url = f"{BUILDBOT}{name}_libretro.so.zip" + cached = self.c.state["cores"].get(name) or {} + + def head() -> dict: + request = urllib.request.Request(url, method="HEAD") + with self.c.host.urlopen(request, timeout=120) as response: + return {"etag": response.headers.get("ETag"), + "last_modified": response.headers.get("Last-Modified")} + stamp = self.retrying(f"{name} nightly", head) + target = self.c.paths.home / "cores" / name / f"{name}_libretro.so" + if (cached.get("etag"), cached.get("last_modified")) != (stamp["etag"], stamp["last_modified"]) \ + or not target.is_file() or file_sha256(target) != cached.get("sha256"): + archive = target.parent / f"{name}_libretro.so.zip" + archive.parent.mkdir(parents=True, exist_ok=True) + + def download() -> None: + with self.c.host.urlopen(url, timeout=600) as response, archive.open("wb") as out: + shutil.copyfileobj(response, out) + self.retrying(f"{name} nightly", download) + safe_extract_so(archive, f"{name}_libretro.so", target) + archive.unlink() + cached = {**stamp, "url": url, "sha256": file_sha256(target), "fetched_at": now_iso()} + self.c.state["cores"][name] = cached + self.c.log(f"core {name}: new nightly {cached['sha256'][:12]} ({stamp['last_modified']})") + repo = profile.get("source") if isinstance(profile.get("source"), str) else None + return Core(name, target, cached["sha256"], artifact=url, + repo=repo if repo and repo.startswith("http") else None) + + def build(self, emulator: str, profile: dict) -> Core: + """The profile's pinned commit, built once and kept.""" + source, commit = profile.get("source"), profile.get("source_commit") + if not (isinstance(source, str) and source.startswith("http") and isinstance(commit, str)): + raise Unavailable("core: not on the buildbot, and no source and source_commit to build") + record = self.c.state["builds"].setdefault(emulator, {}) + if record.get("commit") == commit and record.get("error"): + raise Unavailable(f"core: build of {commit[:12]} failed before: {record['error']}") + out = self.c.paths.home / "builds" / emulator / commit + built = sorted(out.glob("*_libretro.so")) + if not built: + if not self.c.settings.network: + raise Unavailable("core: not on the buildbot, offline, nothing built yet") + try: + built = [self.compile(source, commit, out)] + except Unavailable as exc: + self.c.state["builds"][emulator] = {"commit": commit, "error": str(exc), "at": now_iso()} + raise + command = record.get("command") or "make" + self.c.state["builds"][emulator] = {"commit": commit, "command": command, + "so": built[0].name, "at": now_iso()} + return Core(built[0].name.removesuffix("_libretro.so"), built[0], file_sha256(built[0]), + repo=source, commit=commit, build_command=command) + + def compile(self, source: str, commit: str, out: Path) -> Path: + src = self.c.paths.home / "src" / slug(source) + log = self.c.paths.logs / "builds" / f"{slug(source)}-{commit[:12]}.log" + log.parent.mkdir(parents=True, exist_ok=True) + + def step(argv: list[str], cwd: Path | None = None, timeout: int = 1800) -> None: + with log.open("a", encoding="utf-8") as handle: + handle.write(f"$ {' '.join(argv)}\n") + handle.flush() + proc = self.c.host.run(argv, cwd=cwd, stdout=handle, stderr=subprocess.STDOUT, + check=False, timeout=timeout) + if proc.returncode != 0: + tail = log.read_text(encoding="utf-8", errors="replace").strip().splitlines()[-3:] + raise Unavailable(f"core: {argv[0]} {argv[1] if len(argv) > 1 else ''} exited " + f"{proc.returncode}: {' | '.join(tail)} (log {log})") + if not (src / ".git").is_dir(): + src.parent.mkdir(parents=True, exist_ok=True) + step(["git", "clone", "--quiet", source, str(src)]) + step(["git", "-C", str(src), "fetch", "--quiet", "origin", commit]) + step(["git", "-C", str(src), "checkout", "--quiet", "--force", commit]) + step(["git", "-C", str(src), "submodule", "update", "--init", "--recursive", "--quiet"]) + makefile = "Makefile.libretro" if (src / "Makefile.libretro").is_file() else "Makefile" + jobs = str(max(1, (os.cpu_count() or 2) // 2)) + command = ["make", "-f", makefile, f"-j{jobs}", "platform=unix"] + step(command, cwd=src, timeout=BUILD_TIMEOUT) + built = sorted(src.rglob("*_libretro.so"), key=lambda p: p.stat().st_mtime) + if not built: + raise Unavailable(f"core: {' '.join(command)} produced no *_libretro.so (log {log})") + out.mkdir(parents=True, exist_ok=True) + shutil.copy2(built[-1], out / built[-1].name) + return out / built[-1].name + + +# -------------------------------------------------------------------------- +# Probe runs +# -------------------------------------------------------------------------- + +@dataclass +class ProbeResult: + status: str + record: dict | None = None + reason: str = "" + log: str = "" + + +@dataclass +class Unit: + """One core of one emulator profile, with its systems' content.""" + + emulator: str + profile: dict + core_name: str | None = None + pending: str = "" + # A game engine or utility profile: run when the buildbot has its core, + # otherwise it is a standalone program. + buildbot_only: bool = False + + +@dataclass +class Pass: + started: str + records: list[str] = field(default_factory=list) + findings: dict[str, int] = field(default_factory=dict) + touched: set[str] = field(default_factory=set) + result: str = "complete" + detail: str = "" + + +class Campaign: + def __init__(self, paths: Paths, settings: Settings | None = None, host: Host | None = None, + probe_script: Path | None = None, lock_path: Path | None = None) -> None: + self.paths = paths + self.settings = settings or Settings() + self.host = host or Host() + self.probe_script = probe_script or ROOT / "scripts" / "core_probe.py" + self.lock_path = lock_path or Path(os.environ.get(autorun.LOCK_ENV) or autorun.DEFAULT_LOCK) + self.state = load_state(paths.state) + self.cores = Cores(self) + self.store = ContentStore(self) + self.content = load_content(paths.content_file) + self.deadline = 0.0 + self.current: Pass | None = None + self.dirty_at_start: set[str] = set() + self._db: dict | None = None + + # -- logging ------------------------------------------------------------ + + def log(self, text: str) -> None: + line = f"{now_iso()} {text}" + path = self.paths.logs / f"{dt.date.today().isoformat()}.log" + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + if self.settings.echo: + print(line, file=sys.stderr) + + # -- profiles and files ------------------------------------------------- + + def profiles(self) -> dict[str, dict]: + out = {} + for path in sorted(self.paths.emulators.glob("*.yml")): + if path.name.startswith("_") or path.stem == "00_example": + continue + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if "emulator" in data: + out[path.stem] = data + return out + + @property + def db(self) -> dict: + if self._db is None: + self._db = json.loads(self.paths.db.read_text(encoding="utf-8")) + return self._db + + def local_file(self, entry: dict) -> Path | None: + """The collection's copy of an entry, by its declared hashes or its name.""" + from common import resolve_local_file + + hint = entry.get("path") if isinstance(entry.get("path"), str) else "" + # The database names files relative to the checkout it describes. + with contextlib.chdir(self.paths.root): + path, status = resolve_local_file(entry, self.db, dest_hint=hint or "") + if not path or status in ("not_found", "hash_mismatch"): + return None + local = Path(path) + local = local if local.is_absolute() else self.paths.root / local + return local if local.is_file() else None + + def variant_files(self, entry: dict, primary: Path) -> list[Path]: + """The .variants/ copies of an entry's file the collection holds.""" + names = [entry["name"], *(entry.get("aliases") or ())] + out = [] + index = self.db.get("indexes", {}).get("by_name", {}) + for name in names: + for sha1 in index.get(name, ()): + path = (self.db["files"].get(sha1) or {}).get("path", "") + if "/.variants/" in path: + local = self.paths.root / path + if local.is_file() and local.resolve() != primary.resolve() and local not in out: + out.append(local) + return out + + # -- guards --------------------------------------------------------------- + + def guard(self) -> None: + """Raise Pause when the pass must stop: budget spent, disk full, host busy.""" + if self.host.clock() >= self.deadline: + raise Pause("budget") + free = self.host.free_gb(self.paths.home if self.paths.home.exists() else self.paths.root) + if free < self.settings.min_free_gb: + raise Pause(f"disk: {free:.0f} GB free, the campaign stops under " + f"{self.settings.min_free_gb:.0f} GB") + waited = 0.0 + while self.host.loadavg()[0] > self.host.max_load: + if waited >= self.settings.max_load_wait: + raise Pause(f"busy host: load stayed above {self.host.max_load:.1f} for " + f"{int(waited)} s") + self.host.sleep(self.host.poll) + waited += self.host.poll + + # -- one core_probe invocation ------------------------------------------- + + def probe(self, argv: list[str], log_name: str) -> ProbeResult: + """core_probe with retries for a crashing core. Never raises for the core's sake.""" + command = [sys.executable, str(self.probe_script), *argv] + log = self.paths.logs / dt.date.today().isoformat() / f"{log_name}.log" + log.parent.mkdir(parents=True, exist_ok=True) + repeats = argv.count("--vary") or 1 + reps = int(argv[argv.index("--repeat") + 1]) if "--repeat" in argv else 1 + budget = self.settings.probe_timeout * repeats * reps + 120 + attempt = busy = 0 + while attempt < CRASH_ATTEMPTS: + self.guard() + try: + proc = self.host.run(command, capture_output=True, text=True, check=False, + timeout=budget, cwd=self.paths.root) + except subprocess.TimeoutExpired: + with log.open("a", encoding="utf-8") as handle: + handle.write(f"$ {' '.join(command)}\nkilled after {budget} s\n") + return ProbeResult(FAILED, reason="timeout", log=str(log)) + with log.open("a", encoding="utf-8") as handle: + handle.write(f"$ {' '.join(command)}\n# exit {proc.returncode}\n" + f"{proc.stderr[-20000:]}\n") + if proc.returncode == 0: + try: + return ProbeResult(DONE, record=json.loads(proc.stdout), log=str(log)) + except ValueError: + return ProbeResult(FAILED, reason="core_probe printed no record", log=str(log)) + if proc.returncode == 3: + # Refused on a busy host: wait for it to settle and run it again. + busy += 1 + if busy >= CRASH_ATTEMPTS: + raise Pause("busy host: core_probe refused to record three times") + continue + attempt += 1 + if proc.returncode == 2: + return ProbeResult(FAILED, reason=f"core_probe refused the run: " + f"{proc.stderr.strip()[-300:]}", log=str(log)) + if "TimeoutExpired" in proc.stderr or "timed out after" in proc.stderr: + return ProbeResult(FAILED, reason="timeout", log=str(log)) + self.log(f"{log_name}: attempt {attempt} exited {proc.returncode}") + return ProbeResult(FAILED, reason=f"crashed {CRASH_ATTEMPTS} times", log=str(log)) + + def probe_args(self, mode: str, core: Core, emulator: str, subject: str, content: Content | None, + load: Path | None) -> list[str]: + argv = [mode, "--core", str(core.path), "--record", str(self.paths.root), + "--emulator", emulator, "--subject", subject, + "--max-load", str(self.host.max_load), "--timeout", str(self.settings.probe_timeout), + "--work-dir", str(self.paths.root / "tmp" / "core_probe")] + if core.artifact: + argv += ["--core-artifact", core.artifact] + if core.repo: + argv += ["--core-repo", core.repo] + if core.commit: + argv += ["--core-commit", core.commit] + if core.build_command: + argv += ["--build-command", core.build_command] + if content and load: + argv += ["--content", str(load), "--content-label", content.name] + return argv + + # -- the checks ------------------------------------------------------------- + + def check_key(self, core: Core, content: Content, check: str) -> str: + return f"{core.name}/{content.system}/{slug(content.name)}/{check}" + + def remember(self, emulator: str, key: str, result: ProbeResult, core: Core | None, + digest: str) -> None: + entry = {"status": result.status, "at": now_iso(), "profile_digest": digest, + "core_sha256": core.sha256 if core else None} + if result.reason: + entry["reason"] = result.reason + if result.log: + entry["log"] = result.log + if result.record: + entry["record"] = result.record["id"] + if self.current: + self.current.records.append(result.record["id"]) + self.current.touched.add(mr.records_path(emulator)) + self.state["checks"].setdefault(emulator, {})[key] = entry + + def pending(self, emulator: str, key: str, reason: str, digest: str = "") -> None: + self.state["checks"].setdefault(emulator, {})[key] = { + "status": PENDING, "reason": reason, "at": now_iso(), "profile_digest": digest} + + def system_entries(self, profile: dict, system: str) -> list[tuple[int, dict]]: + from validation import filter_files_by_mode + + libretro = filter_files_by_mode(profile.get("files") or [], standalone=False) + keep = {id(e) for e in libretro} + return [(i, e) for i, e in enumerate(profile.get("files") or []) + if id(e) in keep and system in entry_systems(e, profile) and is_bios(e)] + + def boot_entry(self, entries: list[tuple[int, dict]], content: Content) -> tuple[int, dict] | None: + held = [(i, e) for i, e in entries if region_matches(e, content.region) and self.local_file(e)] + if content.boot_file: + named = [(i, e) for i, e in held + if content.boot_file in (e["name"], *(e.get("aliases") or ()))] + if named: + return named[0] + held.sort(key=lambda item: (item[1].get("priority", 1 << 30), item[0])) + return held[0] if held else None + + def run_unit_content(self, unit: Unit, core: Core, content: Content, digest: str) -> None: + emulator, profile = unit.emulator, unit.profile + checks = self.state["checks"].setdefault(emulator, {}) + entries = self.system_entries(profile, content.system) + boot = self.boot_entry(entries, content) + keys = {name: self.check_key(core, content, name) for name in ("files", "bench")} + if boot: + keys["boot"] = self.check_key(core, content, f"boot:{boot[1]['name']}") + todo = [k for k in keys.values() if should_run(checks.get(k), core.sha256, digest)] + if not todo: + return + try: + folder = self.store.get(content) + load = load_file(folder, content.load) + except Unavailable as exc: + for key in todo: + self.pending(emulator, key, str(exc), digest) + return + frames = ["--frames", str(content.frames), "--frame-hash-every", str(content.frame_hash_every)] + stem = f"{core.name}-{slug(content.name)}" + + # files: every declared file the collection holds, at its place. + staged: dict[str, tuple[int, dict, Path]] = {} + for index, entry in entries: + local = self.local_file(entry) + if local and entry_dest(entry) not in staged: + staged[entry_dest(entry)] = (index, entry, local) + files_record = None + if keys["files"] in todo: + argv = self.probe_args("run", core, emulator, slug(f"files-{stem}"), content, load) + frames + for dest, (_, _, local) in sorted(staged.items()): + argv += ["--system-file", f"{dest}={local}"] + result = self.probe(argv, f"{emulator}-{stem}-files") + self.remember(emulator, keys["files"], result, core, digest) + files_record = result.record + elif checks.get(keys["files"], {}).get("status") == DONE: + files_record = self.read_record(emulator, checks[keys["files"]].get("record")) + + boot_record = None + if boot and keys["boot"] in todo: + index, entry = boot + local = self.local_file(entry) + dest = entry_dest(entry) + argv = self.probe_args("check", core, emulator, slug(f"boot-{entry['name']}-{stem}"), + content, load) + frames + argv += ["--vary", f"{dest}=", "--vary", f"{dest}={local}", "--observable", "frame_hashes", + "--repeat", str(self.settings.check_repeat), + "--claim", f"{core.name} reaches the game on {content.name} without {entry['name']}", + "--method", (f"Two variants, {entry['name']} absent and {entry['name']} as the " + f"collection holds it, each run {self.settings.check_repeat} times " + f"from a fresh process for {content.frames} frames with no button " + f"pressed. Every {content.frame_hash_every}th frame is hashed. The " + f"runs without the file are compared with the runs with it on the " + f"frames that are not one colour.")] + result = self.probe(argv, f"{emulator}-{stem}-boot") + self.remember(emulator, keys["boot"], result, core, digest) + boot_record = result.record + if boot_record: + absent, present = (v["label"] for v in boot_record["variants"]) + verdicts = rules.rule_bios_need(rules.boot_outcome(boot_record, absent, present), + entry, boot_record["id"]) + self.settle(emulator, content, core, index, entry, verdicts) + self.follow_aliases(unit, core, content, load, frames, digest, index, entry, + local, boot_record, absent) + + if files_record and keys["files"] in todo: + self.read_files(unit, core, content, load, frames, digest, staged, files_record) + + if keys["bench"] in todo: + argv = self.probe_args("bench", core, emulator, slug(f"bench-{stem}"), content, load) + argv += ["--warmup", str(self.settings.bench_warmup), "--frames", + str(self.settings.bench_frames), "--repeat", str(self.settings.bench_repeat), + "--no-trace"] + if boot: + local = self.local_file(boot[1]) + argv += ["--system-file", f"{entry_dest(boot[1])}={local}"] + result = self.probe(argv, f"{emulator}-{stem}-bench") + self.remember(emulator, keys["bench"], result, core, digest) + + def read_record(self, emulator: str, rid: str | None) -> dict | None: + if not rid: + return None + path = self.paths.root / mr.records_path(emulator) + if not path.is_file(): + return None + for line in path.read_text(encoding="utf-8").splitlines(): + if f'"id":"{rid}"' in line: + return json.loads(line) + return None + + def read_files(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str], + digest: str, staged: dict[str, tuple[int, dict, Path]], record: dict) -> None: + """Which staged files the core opened: path follow-ups, variants, flags.""" + emulator = unit.emulator + rep = record["variants"][0]["repetitions"][0] + if rep.get("files_opened") is None: + return + opened = {item["path"] for item in rep.get("files_opened") or ()} + probed = list(rep.get("files_probed_absent") or ()) + for dest, (index, entry, local) in sorted(staged.items()): + if dest in opened: + variants = self.variant_files(entry, local) + if variants: + self.run_variants(unit, core, content, load, frames, digest, index, entry, dest, + local, variants) + continue + if not region_matches(entry, content.region): + continue + # A fallback is read only when the files before it are absent: + # staged alone, it shows whether the core reads it at all. + alone = self.follow_up(unit, core, content, load, frames, digest, entry, local, dest, + f"alone:{entry['name']}") + if alone is None: + continue + if dest in self.opened(alone, 1): + continue + probed_alone = [p for r in alone["variants"][0]["repetitions"] + for p in r.get("files_probed_absent") or ()] + others = [p for p in dict.fromkeys(probed + probed_alone) if p != dest + and p.rsplit("/", 1)[-1].casefold() == dest.rsplit("/", 1)[-1].casefold()] + moved = False + for candidate in others[:MAX_ALIAS_CANDIDATES]: + record_c = self.follow_up(unit, core, content, load, frames, digest, entry, local, + candidate, f"path:{entry['name']}:{candidate}") + if record_c and candidate in self.opened(record_c, 1): + verdict = rules.rule_path(entry, dest, rules.Opened(candidate, record_c["id"])) + if verdict: + self.settle(emulator, content, core, index, entry, [verdict]) + moved = True + break + if not moved: + verdict = rules.rule_never_opened(entry, [record["id"], alone["id"]], content.name) + self.settle(emulator, content, core, index, entry, [verdict]) + + @staticmethod + def opened(record: dict, position: int) -> set[str]: + out: set[str] = set() + for rep in record["variants"][position]["repetitions"]: + out |= {item["path"] for item in rep.get("files_opened") or ()} + return out + + def follow_up(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str], + digest: str, entry: dict, local: Path, dest: str, check: str) -> dict | None: + """The file staged at dest alone, against nothing there.""" + key = self.check_key(core, content, check) + previous = self.state["checks"].get(unit.emulator, {}).get(key) + if not should_run(previous, core.sha256, digest): + return self.read_record(unit.emulator, (previous or {}).get("record")) + stem = f"{core.name}-{slug(content.name)}" + argv = self.probe_args("check", core, unit.emulator, slug(f"{check}-{stem}"), content, load) + argv += frames + ["--vary", f"{dest}=", "--vary", f"{dest}={local}", + "--observable", "frame_hashes", "--repeat", "1", + "--claim", f"{core.name} opens {entry['name']} as {dest}", + "--method", (f"{entry['name']} staged as {dest}, against nothing there, " + f"{content.frames} frames of {content.name}. The files the " + f"core opened are read from strace.")] + result = self.probe(argv, f"{unit.emulator}-{stem}-{slug(check)}") + self.remember(unit.emulator, key, result, core, digest) + return result.record + + def follow_aliases(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str], + digest: str, index: int, entry: dict, local: Path, record: dict, + absent_label: str) -> None: + """Names the core looked for, with the file absent, that no entry declares.""" + declared = {e["name"] for e in unit.profile.get("files") or ()} + declared |= {a for e in unit.profile.get("files") or () for a in e.get("aliases") or ()} + suffix = Path(entry["name"]).suffix.casefold() + dest = entry_dest(entry) + folder = dest.rsplit("/", 1)[0] if "/" in dest else "" + probed: list[str] = [] + for rep in rules.variant(record, absent_label)["repetitions"]: + for path in rep.get("files_probed_absent") or (): + name = path.rsplit("/", 1)[-1] + where = path.rsplit("/", 1)[0] if "/" in path else "" + if (where == folder and name not in declared and Path(name).suffix.casefold() == suffix + and path not in probed): + probed.append(path) + for candidate in probed[:MAX_ALIAS_CANDIDATES]: + found = self.follow_up(unit, core, content, load, frames, digest, entry, local, candidate, + f"alias:{entry['name']}:{candidate.rsplit('/', 1)[-1]}") + if found and candidate in self.opened(found, 1): + verdict = rules.rule_alias(self.reload_entry(unit, index), rules.Opened(candidate, found["id"])) + if verdict: + self.settle(unit.emulator, content, core, index, self.reload_entry(unit, index), [verdict]) + + def run_variants(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str], + digest: str, index: int, entry: dict, dest: str, primary: Path, + variants: list[Path]) -> None: + key = self.check_key(core, content, f"variants:{entry['name']}") + if not should_run(self.state["checks"].get(unit.emulator, {}).get(key), core.sha256, digest): + return + stem = f"{core.name}-{slug(content.name)}" + argv = self.probe_args("check", core, unit.emulator, slug(f"variants-{entry['name']}-{stem}"), + content, load) + frames + for path in [primary, *variants]: + argv += ["--vary", f"{dest}={path}"] + argv += ["--observable", "frame_hashes", "--repeat", str(self.settings.check_repeat), + "--claim", f"the copies of {entry['name']} change what {core.name} shows on {content.name}", + "--method", (f"Each copy the collection holds staged as {dest}, each run " + f"{self.settings.check_repeat} times for {content.frames} frames, " + f"compared on the sampled frame hashes.")] + result = self.probe(argv, f"{unit.emulator}-{stem}-variants") + self.remember(unit.emulator, key, result, core, digest) + if result.record: + status = result.record["check"]["result"]["status"] + verdict = rules.rule_variants(status, [v["label"] for v in result.record["variants"]], + entry, result.record["id"]) + if verdict: + self.settle(unit.emulator, content, core, index, entry, [verdict]) + + # -- verdicts ----------------------------------------------------------------- + + def reload_entry(self, unit: Unit, index: int) -> dict: + path = self.paths.emulators / f"{unit.emulator}.yml" + unit.profile = yaml.safe_load(path.read_text(encoding="utf-8")) + return unit.profile["files"][index] + + def settle(self, emulator: str, content: Content, core: Core, index: int, entry: dict, + verdicts: Sequence[rules.Verdict]) -> None: + """Apply what the table settles, queue the rest, resolve what now agrees.""" + path = self.paths.emulators / f"{emulator}.yml" + for verdict in verdicts: + fresh = (yaml.safe_load(path.read_text(encoding="utf-8")).get("files") or [])[index] + if fresh.get("name") != entry.get("name"): + self.log(f"{emulator}: files[{index}] is no longer {entry.get('name')}, nothing applied") + return + says = rules.current(fresh, verdict.field) if verdict.field in rules.SETTLED_FIELDS \ + else None + agrees = verdict.field in rules.SETTLED_FIELDS and rules.agrees(fresh, verdict) + if verdict.apply: + updates: dict[str, object] = {} + if not agrees or verdict.field not in fresh: + updates[verdict.field] = verdict.value + evidence = rules.merge_evidence(fresh.get("evidence"), verdict) + if evidence != fresh.get("evidence"): + updates["evidence"] = evidence + if updates: + try: + changed = campaign_profiles.write_fields(path, index, updates) + except campaign_profiles.ProfileEditError as exc: + self.finding(emulator, content, core, fresh, verdict, "open", says, + extra=f". Not written: {exc}") + continue + if changed and self.current: + self.current.touched.add(path.relative_to(self.paths.root).as_posix()) + if agrees: + self.resolve(emulator, content, core, fresh, verdict, says) + else: + self.finding(emulator, content, core, fresh, verdict, "applied", says) + continue + if verdict.rule == "never_opened": + self.finding(emulator, content, core, fresh, verdict, "flagged", None) + elif agrees: + self.resolve(emulator, content, core, fresh, verdict, says) + else: + self.finding(emulator, content, core, fresh, verdict, "open", says) + + def finding_key(self, emulator: str, content: Content, entry: dict, verdict: rules.Verdict) -> str: + return f"{emulator}/{content.system}/{entry['name']}/{verdict.field}" + + def latest_findings(self) -> dict[str, dict]: + latest: dict[str, dict] = {} + for line in mr.read_campaign(self.paths.root, mr.FINDINGS): + latest[line["key"]] = line + return latest + + def finding(self, emulator: str, content: Content, core: Core, entry: dict, + verdict: rules.Verdict, status: str, says: object, extra: str = "") -> None: + key = self.finding_key(emulator, content, entry, verdict) + previous = self.latest_findings().get(key) + if previous and previous["status"] == status and previous["rule"] == verdict.rule \ + and previous["run_shows"] == verdict.value and previous["profile_says"] == says: + return + line = mr.seal_line({ + "schema_version": 1, "found_at": now_iso(), "key": key, "emulator": emulator, + "system": content.system, "file": entry["name"], "field": verdict.field, + "status": status, "rule": verdict.rule, "profile_says": says, "run_shows": verdict.value, + "statement": verdict.statement + extra, "evidence": list(verdict.evidence), + "core": {"file": core.path.name, "sha256": core.sha256, + "library_version": self.library_version(emulator, verdict.evidence)}, + "content": content.name, + }, "found_at") + path = self.paths.root / mr.records_path(mr.FINDINGS) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(mr.line_text(line) + "\n") + if self.current: + self.current.findings[status] = self.current.findings.get(status, 0) + 1 + self.current.touched.add(mr.records_path(mr.FINDINGS)) + self.log(f"finding {status}: {key}: {verdict.statement}") + + def resolve(self, emulator: str, content: Content, core: Core, entry: dict, + verdict: rules.Verdict, says: object) -> None: + previous = self.latest_findings().get(self.finding_key(emulator, content, entry, verdict)) + if previous and previous["status"] in ("open", "flagged"): + self.finding(emulator, content, core, entry, verdict, "resolved", says) + + def library_version(self, emulator: str, ids: Sequence[str]) -> str | None: + for rid in ids: + record = self.read_record(emulator, rid) + if record: + return record["core"].get("library_version") + return None + + # -- the pass ----------------------------------------------------------------- + + def units(self, only: set[str], system: str | None) -> list[Unit]: + out = [] + for emulator, profile in sorted(self.profiles().items()): + cores = {str(c) for c in profile.get("cores") or ()} + if only and emulator not in only and not (cores & only): + continue + systems = set(profile.get("systems") or ()) + if system and system not in systems: + continue + kind = str(profile.get("type") or "") + if kind in ("alias", "launcher", "test"): + continue + if "libretro" in kind or kind in ("game", "utility"): + out.append(Unit(emulator, profile, buildbot_only="libretro" not in kind)) + continue + out.append(Unit(emulator, profile, pending="standalone")) + return out + + def unit_contents(self, unit: Unit, system: str | None) -> list[Content]: + systems = [s for s in unit.profile.get("systems") or () if not system or s == system] + return [c for s in systems for c in self.content.get(s, ())] + + def run(self, only: set[str], system: str | None, dry_run: bool = False, + commit_every: int = 0, commit: bool = False) -> Pass: + self.deadline = self.host.clock() + self.settings.budget + current = self.current = Pass(started=now_iso()) + units = self.units(only, system) + if dry_run: + self.describe(units, system) + return current + for unit in units: + for content in self.unit_contents(unit, system): + self.store.hold(content, unit.emulator) + done_units = 0 + try: + for unit in units: + self.run_unit(unit, system) + done_units += 1 + save_state(self.paths.state, self.state) + if commit and commit_every and done_units % commit_every == 0: + self.commit_batch() + except Pause as exc: + current.result = "budget" if str(exc) == "budget" else "paused" + current.detail = str(exc) + self.log(f"pass paused: {exc}") + except KeyboardInterrupt: + current.result, current.detail = "stopped", "interrupted" + finally: + for unit in units: + for content in self.unit_contents(unit, system): + self.store.release(content, unit.emulator) + self.prune_logs() + self.write_run_line(current) + save_state(self.paths.state, self.state) + if commit: + self.commit_batch() + return current + + def run_unit(self, unit: Unit, system: str | None) -> None: + emulator, profile = unit.emulator, unit.profile + digest = profile_digest(profile) + contents = self.unit_contents(unit, system) + systems = [s for s in profile.get("systems") or () if not system or s == system] + if unit.pending: + for s in systems: + self.pending(emulator, f"*/{s}/*", unit.pending, digest) + return + if not contents: + for s in systems: + self.pending(emulator, f"*/{s}/*", f"content: no test content listed for {s}", digest) + return + try: + cores = self.unit_cores(emulator, profile, unit.buildbot_only) + except Unavailable as exc: + for s in systems: + self.pending(emulator, f"*/{s}/*", str(exc), digest) + self.log(f"{emulator}: {exc}") + return + for s in systems: + self.state["checks"].setdefault(emulator, {}).pop(f"*/{s}/*", None) + for core in cores: + self.log(f"{emulator}: core {core.name} {core.sha256[:12]}") + for content in contents: + self.guard() + self.run_unit_content(Unit(emulator, profile, core.name), core, content, digest) + for content in contents: + self.store.release(content, emulator) + self.state["emulators"][emulator] = {"profile_digest": profile_digest( + yaml.safe_load((self.paths.emulators / f"{emulator}.yml").read_text(encoding="utf-8"))), + "at": now_iso()} + # The campaign's own edits do not make its checks run again. + new_digest = self.state["emulators"][emulator]["profile_digest"] + for entry in self.state["checks"].get(emulator, {}).values(): + if entry.get("profile_digest") == digest: + entry["profile_digest"] = new_digest + + def unit_cores(self, emulator: str, profile: dict, buildbot_only: bool = False) -> list[Core]: + names = self.cores.names(profile) + if names: + return [self.cores.nightly(name, profile) for name in names] + if buildbot_only: + raise Unavailable("standalone: no libretro core on the buildbot") + return [self.cores.build(emulator, profile)] + + def describe(self, units: list[Unit], system: str | None) -> None: + for unit in units: + contents = self.unit_contents(unit, system) + if unit.pending: + print(f"{unit.emulator}: pending, {unit.pending}") + elif not contents: + print(f"{unit.emulator}: pending, no test content for " + f"{', '.join(unit.profile.get('systems') or ())}") + else: + cores = sorted({str(c) for c in unit.profile.get("cores") or ()}) + print(f"{unit.emulator}: cores {', '.join(cores)} on " + f"{', '.join(c.name for c in contents)}") + + def counts(self) -> tuple[dict[str, int], dict[str, int], list[dict]]: + counts = {DONE: 0, FAILED: 0, PENDING: 0} + reasons: dict[str, int] = {} + failed = [] + for emulator, checks in sorted(self.state["checks"].items()): + for key, entry in sorted(checks.items()): + status = entry.get("status", PENDING) + counts[status] = counts.get(status, 0) + 1 + if status == PENDING: + reason = reason_class(entry.get("reason", "")) + reasons[reason] = reasons.get(reason, 0) + 1 + elif status == FAILED: + failed.append({"emulator": emulator, "check": key, "reason": entry.get("reason", "")}) + return counts, reasons, failed + + def write_run_line(self, current: Pass) -> None: + counts, reasons, failed = self.counts() + line = mr.seal_line({ + "schema_version": 1, "started_at": current.started, "ended_at": now_iso(), + "result": current.result, "detail": current.detail, "counts": counts, + "pending_reasons": reasons, "failed": failed, "records": current.records, + "findings": dict(sorted(current.findings.items())), + }, "started_at") + path = self.paths.root / mr.records_path(mr.CAMPAIGN_RUNS) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(mr.line_text(line) + "\n") + current.touched.add(mr.records_path(mr.CAMPAIGN_RUNS)) + self.state["last_run"] = {k: line[k] for k in ("id", "started_at", "ended_at", "result", "detail")} + + def prune_logs(self) -> None: + limit = time.time() - LOG_DAYS * 86400 + for path in self.paths.logs.rglob("*.log") if self.paths.logs.is_dir() else (): + if path.stat().st_mtime < limit: + path.unlink() + + # -- commits (direct runs, autorun commits by its own rules) ------------------- + + def plan_commits(self, dirty_at_start: set[str]) -> list[tuple[str, list[str]]]: + """(subject, paths) of each commit: the records, then one per profile.""" + touched = sorted(self.current.touched if self.current else ()) + records = [p for p in touched if p.startswith(f"{mr.RECORDS_DIR}/") and p not in dirty_at_start] + plan = [] + if records: + plan.append(("chore: add campaign measurements", records)) + for path in touched: + if path.startswith("emulators/") and path not in dirty_at_start: + subject = f"chore: apply tested fields to {Path(path).stem}" + if len(subject) > 50: + subject = "chore: apply tested profile fields" + plan.append((subject, [path])) + return plan + + def commit_batch(self, dry_run: bool = False) -> list[str]: + plan = self.plan_commits(self.dirty_at_start) + if dry_run or not plan: + for subject, paths in plan: + print(f"would commit {subject!r}: {', '.join(paths)}") + return [] + if gate(self.paths.root, [Path(p).stem for _, ps in plan for p in ps if p.startswith("emulators/")], + self.host) != 0: + self.log("gate failed, nothing committed") + return [] + made = [] + for subject, paths in plan: + problems = autorun.check_subject(subject) + if problems: + raise ValueError(f"{subject!r}: {', '.join(problems)}") + made.append(autorun.commit_paths(self.paths.root, paths, autorun.CommitRule(subject, tuple(paths)))) + if self.current: + self.current.touched.clear() + return made + + +def reason_class(reason: str) -> str: + """A pending reason without its detail, to count them.""" + head = reason.split(":", 1)[0].strip() if ":" in reason else reason + return head or "unknown" + + +# -------------------------------------------------------------------------- +# Gate +# -------------------------------------------------------------------------- + +def gate(root: Path, emulators: Sequence[str], host: Host | None = None) -> int: + """The measurement tests, the schemas, and verify plus an emulator pack + for each profile the campaign changed.""" + host = host or Host() + python = sys.executable + steps = [ + [python, "-m", "unittest", "tests.test_measurements", "tests.test_campaign"], + [python, "scripts/validate_schemas.py", "--source-only"], + ] + for emulator in sorted(set(emulators)): + steps.append([python, "scripts/verify.py", "--emulator", emulator]) + steps.append([python, "scripts/generate_pack.py", "--emulator", emulator, "--offline", + "--output-dir", "tmp/campaign/packs"]) + for argv in steps: + proc = host.run(argv, cwd=root, check=False) + if proc.returncode != 0: + print(f"gate: {' '.join(argv)} exited {proc.returncode}", file=sys.stderr) + return 1 + shutil.rmtree(root / "tmp" / "campaign" / "packs", ignore_errors=True) + return 0 + + +def changed_profiles(root: Path) -> list[str]: + return [Path(p).stem for p in autorun.changed_files(root) + if p.startswith("emulators/") and p.endswith(".yml")] + + +# -------------------------------------------------------------------------- +# Reports +# -------------------------------------------------------------------------- + +def status_report(campaign: Campaign) -> dict: + counts, reasons, failed = campaign.counts() + runs = mr.read_campaign(campaign.paths.root, mr.CAMPAIGN_RUNS) + per = {} + for emulator, checks in sorted(campaign.state["checks"].items()): + per[emulator] = {key: {k: v for k, v in entry.items() if k in ("status", "reason", "record", "at")} + for key, entry in sorted(checks.items())} + return {"last_run": runs[-1] if runs else None, "counts": counts, "pending_reasons": reasons, + "failed": failed, "emulators": per} + + +def render_status(report: dict) -> str: + lines = [] + last = report["last_run"] + if last: + lines.append(f"last run {last['started_at']} to {last['ended_at']}: {last['result']}" + + (f" ({last['detail']})" if last["detail"] else "")) + else: + lines.append("no run yet") + counts = report["counts"] + lines.append(f"{counts.get(DONE, 0)} done, {counts.get(FAILED, 0)} failed, " + f"{counts.get(PENDING, 0)} pending") + for reason, number in sorted(report["pending_reasons"].items(), key=lambda kv: -kv[1]): + lines.append(f" pending {number:5} {reason}") + for item in report["failed"]: + lines.append(f" FAILED {item['emulator']} {item['check']}: {item['reason']}") + for emulator, checks in report["emulators"].items(): + ran = [k for k, v in checks.items() if v["status"] != PENDING] + if ran: + lines.append(f"{emulator}") + for key in ran: + entry = checks[key] + lines.append(f" {entry['status']:7} {key}" + (f" {entry['record']}" if entry.get("record") else "") + + (f" ({entry['reason']})" if entry.get("reason") else "")) + return "\n".join(lines) + + +def open_findings(root: Path, everything: bool = False) -> list[dict]: + latest: dict[str, dict] = {} + for line in mr.read_campaign(root, mr.FINDINGS): + latest[line["key"]] = line + return [line for key, line in sorted(latest.items()) + if everything or line["status"] in ("open", "flagged")] + + +# -------------------------------------------------------------------------- +# Command line +# -------------------------------------------------------------------------- + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + run_p = sub.add_parser("run", help="one pass: cores, content, checks, rules") + run_p.add_argument("--only", default="", help="emulator profiles or core names, comma-separated") + run_p.add_argument("--system", help="one system id") + run_p.add_argument("--budget", type=parse_budget, default=6 * 3600.0, + help="stop starting checks after this long (6h, 90m, 3600s)") + run_p.add_argument("--max-load-wait", type=float, default=3600, + help="seconds to wait for an idle host before pausing the pass") + run_p.add_argument("--min-free-gb", type=float, default=30, help="pause below this free space") + run_p.add_argument("--cache-cap-gb", type=float, default=20, + help="content the campaign fetched is evicted past this size, oldest use first") + run_p.add_argument("--romfetch", default="", + help=f"romfetch command (default: ${ROMFETCH_ENV}, else romfetch on PATH)") + run_p.add_argument("--offline", action="store_true", help="no buildbot, no romfetch, no build") + run_p.add_argument("--dry-run", action="store_true", help="print the plan, run nothing") + run_p.add_argument("--commit", action="store_true", + help="commit records, findings and each changed profile behind the gate") + run_p.add_argument("--commit-every", type=int, default=0, metavar="N", + help="with --commit, also commit after every N emulators") + status_p = sub.add_parser("status", help="done, failed and pending checks, and why") + status_p.add_argument("--json", action="store_true") + findings_p = sub.add_parser("findings", help="profile claims the runs contradict, open for review") + findings_p.add_argument("--all", action="store_true", help="applied and resolved too") + findings_p.add_argument("--json", action="store_true") + sub.add_parser("gate", help="tests, schemas, verify and pack of the changed profiles") + timer_p = sub.add_parser("install-timer", + help="write the autorun units, the campaign job's timer among them") + timer_p.add_argument("--enable", action="store_true") + args = parser.parse_args(argv) + + paths = Paths.default() + if args.command == "gate": + return gate(paths.root, changed_profiles(paths.root)) + if args.command == "install-timer": + command = [sys.executable, str(ROOT / "scripts" / "autorun.py"), "install-timers"] + return subprocess.run(command + (["--enable"] if args.enable else []), check=False).returncode + if args.command == "findings": + lines = open_findings(paths.root, args.all) + if args.json: + print(json.dumps(lines, indent=2)) + for line in [] if args.json else lines: + print(f"{line['status']:8} {line['key']}: profile says {json.dumps(line['profile_says'])}, " + f"run shows {json.dumps(line['run_shows'])}. {line['statement']} " + f"[{', '.join(line['evidence'])}]") + return 0 + settings = Settings() + if args.command == "run": + settings = Settings(budget=args.budget, max_load_wait=args.max_load_wait, + min_free_gb=args.min_free_gb, cache_cap_gb=args.cache_cap_gb, + romfetch=args.romfetch, network=not args.offline) + campaign = Campaign(paths, settings) + if args.command == "status": + report = status_report(campaign) + print(json.dumps(report, indent=2) if args.json else render_status(report)) + return 0 + only = {item.strip() for item in args.only.split(",") if item.strip()} + if args.dry_run: + campaign.run(only, args.system, dry_run=True) + return 0 + campaign.dirty_at_start = set(autorun.changed_files(paths.root)) if args.commit else set() + try: + with host_lock(campaign.lock_path): + current = campaign.run(only, args.system, commit_every=args.commit_every, + commit=args.commit) + except autorun.LockBusy as exc: + print(f"campaign: deferred, {exc}", file=sys.stderr) + return 75 + counts, _, _ = campaign.counts() + print(json.dumps({"result": current.result, "detail": current.detail, "counts": counts, + "records": len(current.records), "findings": current.findings})) + return 75 if current.result == "paused" else 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/scripts/campaign_profiles.py b/scripts/campaign_profiles.py new file mode 100644 index 000000000..0df90652a --- /dev/null +++ b/scripts/campaign_profiles.py @@ -0,0 +1,161 @@ +"""Write a settled field into one entry of a profile, and nothing else. + +The profile is edited as text, so its comments, order and quoting stay as +the maintainer wrote them. Every edit is checked by parsing the result: the +edited entry must equal the old one with the new fields, and every other +part of the profile must parse to what it did before. An edit that cannot +hold both is refused (ProfileEditError), never written. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import yaml + +ITEM = re.compile(r"^(?P\s*)- ") +KEY = re.compile(r"^(?P\s*)(?P[A-Za-z_][A-Za-z0-9_]*):(?P.*)$") +# Where a new key goes: after the first of these the entry has, else last. +ANCHORS = { + "required": ("name",), + "hle_fallback": ("required", "name"), + "path": ("name",), + "aliases": ("path", "name"), +} + + +class ProfileEditError(ValueError): + """An edit the profile's layout does not allow to make safely.""" + + +def scalar(value: object) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if value is None: + return "null" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + return json.dumps(value, ensure_ascii=False) + if isinstance(value, list): + return "[" + ", ".join(scalar(v) for v in value) + "]" + raise ProfileEditError(f"no scalar form for {value!r}") + + +def item_spans(lines: list[str]) -> list[tuple[int, int, int]]: + """(first line, end line, item indent) of each item of the top-level files: list.""" + try: + start = next(i for i, line in enumerate(lines) if line.rstrip() == "files:") + except StopIteration as exc: + raise ProfileEditError("no top-level files: list") from exc + spans: list[tuple[int, int, int]] = [] + indent: int | None = None + current: int | None = None + end = len(lines) + for i in range(start + 1, len(lines)): + line = lines[i] + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + width = len(line) - len(line.lstrip(" ")) + if width == 0 and not line.startswith("-"): + end = i + break + match = ITEM.match(line) + if match and (indent is None or width == indent): + indent = width + if current is not None: + spans.append((current, i, indent)) + current = i + if current is not None: + spans.append((current, end, indent or 0)) + # An item ends at its last line of content: the comments and blank lines + # before the next item belong to that item. + trimmed = [] + for first, stop, width in spans: + last = stop + while last > first + 1 and (not lines[last - 1].strip() or lines[last - 1].strip().startswith("#")): + last -= 1 + trimmed.append((first, last, width)) + return trimmed + + +def key_lines(lines: list[str], first: int, stop: int, key_indent: int) -> dict[str, tuple[int, int]]: + """key -> (line, end line) for each key of one item, its continuation lines included.""" + keys: dict[str, tuple[int, int]] = {} + order: list[tuple[str, int]] = [] + for i in range(first, stop): + # The item's first line carries the dash where its first key's indent is. + line = " " * key_indent + lines[i][key_indent:] if i == first else lines[i] + match = KEY.match(line) + if match and len(match["indent"]) == key_indent: + order.append((match["key"], i)) + for position, (key, line) in enumerate(order): + end = order[position + 1][1] if position + 1 < len(order) else stop + while end > line + 1 and (not lines[end - 1].strip() or lines[end - 1].strip().startswith("#")): + end -= 1 + keys[key] = (line, end) + return keys + + +def _render(key: str, value: object, indent: int, block: bool = False) -> list[str]: + """One key as lines. A list keeps the block style it was written in.""" + pad = " " * indent + if isinstance(value, dict): + return [f"{pad}{key}:"] + [f"{pad} {k}: {scalar(v)}" for k, v in value.items()] + if block and isinstance(value, list): + return [f"{pad}{key}:"] + [f"{pad} - {scalar(v)}" for v in value] + return [f"{pad}{key}: {scalar(value)}"] + + +def set_fields(text: str, index: int, updates: dict[str, object]) -> str: + """The profile with these keys of files[index] set, the rest untouched.""" + before = yaml.safe_load(text) + entries = before.get("files") or [] + if not 0 <= index < len(entries): + raise ProfileEditError(f"no files[{index}]") + lines = text.splitlines() + spans = item_spans(lines) + if len(spans) != len(entries): + raise ProfileEditError(f"{len(spans)} items read as text, {len(entries)} parsed") + first, stop, item_indent = spans[index] + key_indent = item_indent + 2 + for key, value in updates.items(): + keys = key_lines(lines, first, stop, key_indent) + if key in keys: + start, end = keys[key] + rendered = _render(key, value, key_indent, block=lines[start].rstrip().endswith(":")) + if start == first: + raise ProfileEditError(f"files[{index}]: {key} is the item's first key") + lines[start:end] = rendered + stop += len(rendered) - (end - start) + continue + rendered = _render(key, value, key_indent) + anchor = next((keys[a][1] for a in ANCHORS.get(key, ()) if a in keys), None) + at = anchor if anchor is not None else stop + lines[at:at] = rendered + stop += len(rendered) + out = "\n".join(lines) + ("\n" if text.endswith("\n") else "") + after = yaml.safe_load(out) + expected = {**entries[index], **updates} + if after.get("files", [None] * (index + 1))[index] != expected: + raise ProfileEditError(f"files[{index}]: the edit does not parse to the intended entry") + rest_before = {k: v for k, v in before.items() if k != "files"} + rest_after = {k: v for k, v in after.items() if k != "files"} + others_before = [e for i, e in enumerate(entries) if i != index] + others_after = [e for i, e in enumerate(after.get("files") or []) if i != index] + if rest_before != rest_after or others_before != others_after: + raise ProfileEditError(f"files[{index}]: the edit changes another part of the profile") + return out + + +def write_fields(path: Path, index: int, updates: dict[str, object]) -> bool: + """Apply the edit to the file. False when it changes nothing.""" + text = path.read_text(encoding="utf-8") + out = set_fields(text, index, updates) + if out == text: + return False + path.write_text(out, encoding="utf-8") + return True diff --git a/scripts/campaign_rules.py b/scripts/campaign_rules.py new file mode 100644 index 000000000..e9da2543e --- /dev/null +++ b/scripts/campaign_rules.py @@ -0,0 +1,203 @@ +"""The campaign's fixed rule table: which observable settles which profile field. + +Each rule is a pure function of a core probe record (or what was read from +one) and a profile entry. It returns verdicts. A verdict with `apply` set is +settled by the table and written into the profile with the record ids as +`evidence:`. Any other verdict goes to the review queue +(measurements/findings.jsonl). + + observable field settled + boots to the game with no file hle_fallback true + no game screen without it, the game with it required true + hle_fallback false + opens the staged file under another path path that path + opens the staged file under another name aliases name added + a staged file the content should use, never read - flagged only + +"Boots to the game" is read from the sampled frames: the run without the +file shows at least MIN_SHARED_SCREENS distinct screens that the run with +the file also shows, single-colour frames excluded. "No game screen" is a +run that did not load, or whose every sampled frame is a single colour. A +run that shows screens of its own that the reference never shows settles +nothing. + +Standard library only. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field + +MIN_SHARED_SCREENS = 2 +BOOTS = "boots" +FAILS = "fails" +INCONCLUSIVE = "inconclusive" + +# A field an entry leaves out reads as this value. +FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": []} +SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases") + + +@dataclass(frozen=True) +class Boot: + """What the runs without the file showed, against the runs with it.""" + + status: str + detail: str + shared: int = 0 + + +@dataclass(frozen=True) +class Verdict: + rule: str + field: str + value: object + statement: str + evidence: tuple[str, ...] + apply: bool = False + + +@dataclass(frozen=True) +class Opened: + """Where a run found the staged file opened.""" + + dest: str + record_id: str + extra: dict = field(default_factory=dict) + + +def screens(repetition: dict) -> set[str]: + """The distinct sampled frames of a repetition that are not one colour.""" + observables = repetition.get("observables") or {} + uniform = set(observables.get("uniform_frames") or ()) + return {digest for _, digest in observables.get("frame_hashes") or () if digest not in uniform} + + +def variant(record: dict, label: str) -> dict: + for item in record.get("variants") or (): + if item.get("label") == label: + return item + raise KeyError(f"record {record.get('id')}: no variant {label!r}") + + +def boot_outcome(record: dict, absent_label: str, present_label: str) -> Boot: + """Whether the core reached the game without the file. + + Every repetition must agree: one that boots and one that does not leave + the question open. + """ + present = variant(record, present_label)["repetitions"] + if any(not rep.get("loaded") for rep in present): + return Boot(INCONCLUSIVE, "with the file, the core did not load the content") + reference: set[str] = set() + for rep in present: + shown = screens(rep) + if not shown: + return Boot(INCONCLUSIVE, "with the file, the core showed no game screen") + reference |= shown + outcomes: list[str] = [] + shared_counts: list[int] = [] + for rep in variant(record, absent_label)["repetitions"]: + if not rep.get("loaded"): + outcomes.append(FAILS) + continue + shown = screens(rep) + if not shown: + outcomes.append(FAILS) + continue + shared = len(shown & reference) + shared_counts.append(shared) + outcomes.append(BOOTS if shared >= MIN_SHARED_SCREENS else INCONCLUSIVE) + shared = min(shared_counts) if shared_counts else 0 + if all(o == BOOTS for o in outcomes): + return Boot(BOOTS, f"without the file, every run showed {shared} or more screens the " + f"run with the file shows", shared) + if all(o == FAILS for o in outcomes): + return Boot(FAILS, "without the file, no run loaded the content or showed more than " + "one colour, while the run with the file reached the game") + return Boot(INCONCLUSIVE, "without the file, the runs showed screens the run with the file " + "never shows, or disagreed with each other", shared) + + +def current(entry: dict, name: str) -> object: + if name in entry: + return entry[name] + return FIELD_DEFAULTS.get(name) + + +def agrees(entry: dict, verdict: Verdict) -> bool: + """True when the profile already says what the run showed.""" + if verdict.field == "aliases": + return set(verdict.value or ()) <= set(entry.get("aliases") or ()) + return current(entry, verdict.field) == verdict.value + + +def rule_bios_need(boot: Boot, entry: dict, record_id: str) -> list[Verdict]: + """Boot with no file against boot with the declared file.""" + name = entry["name"] + evidence = (record_id,) + if boot.status == BOOTS: + out = [Verdict("boots_without_file", "hle_fallback", True, + f"the core reaches the game without {name}: {boot.detail}", evidence, True)] + if entry.get("required") is True: + out.append(Verdict( + "boots_without_file", "required", False, + f"the profile says {name} is required, and the core reaches the game " + f"without it. The table settles hle_fallback from this run, not required", + evidence)) + return out + if boot.status == FAILS: + return [ + Verdict("fails_without_file", "required", True, + f"the core reaches the game only with {name}: {boot.detail}", evidence, True), + Verdict("fails_without_file", "hle_fallback", False, + f"no stand-in for {name}: {boot.detail}", evidence, True), + ] + return [Verdict("inconclusive", "hle_fallback", None, + f"the runs with and without {name} settle nothing: {boot.detail}", evidence)] + + +def rule_path(entry: dict, declared: str, opened: Opened) -> Verdict | None: + """The core opened the file at another place under the system directory.""" + if opened.dest == declared: + return None + return Verdict("opened_under_other_path", "path", opened.dest, + f"the core opened {entry['name']} as {opened.dest}, the profile declares " + f"{declared}", (opened.record_id,), True) + + +def rule_alias(entry: dict, opened: Opened) -> Verdict | None: + """The core opened the file under a name the entry does not declare.""" + name = opened.dest.rsplit("/", 1)[-1] + known = [entry["name"], *(entry.get("aliases") or ())] + if name in known: + return None + return Verdict("opened_other_name", "aliases", [*(entry.get("aliases") or ()), name], + f"the core opened {entry['name']} under the name {name}", + (opened.record_id,), True) + + +def rule_never_opened(entry: dict, record_ids: Sequence[str], content: str) -> Verdict: + return Verdict("never_opened", "opened", False, + f"{entry['name']} was staged and the core never opened it while running " + f"{content}", tuple(record_ids)) + + +def rule_variants(status: str, labels: Iterable[str], entry: dict, record_id: str) -> Verdict | None: + """Copies of one file that change what the core shows: a pack choice to review.""" + if status != "differs": + return None + return Verdict("variant_changes_output", "variants", sorted(labels), + f"the copies of {entry['name']} change what the core shows", + (record_id,)) + + +def merge_evidence(existing: dict | None, verdict: Verdict) -> dict: + """The entry's evidence: map with the verdict's record ids added under its field.""" + out = {key: list(value) for key, value in (existing or {}).items()} + ids = out.setdefault(verdict.field, []) + for rid in verdict.evidence: + if rid not in ids: + ids.append(rid) + return {key: out[key] for key in sorted(out)} diff --git a/scripts/core_probe.py b/scripts/core_probe.py index e900d979f..846efd602 100644 --- a/scripts/core_probe.py +++ b/scripts/core_probe.py @@ -269,6 +269,14 @@ def render_printf(fmt: bytes, args: Sequence[int]) -> str: return "".join(out).rstrip("\n") +def uniform_frame(frame: bytes, pixel_bytes: int) -> bool: + """True for a frame of one colour, an empty one included: a screen that + shows nothing of the game.""" + if len(frame) <= pixel_bytes: + return True + return frame == frame[:pixel_bytes] * (len(frame) // pixel_bytes) + + def png_bytes(width: int, height: int, rgb: bytes) -> bytes: def chunk(tag: bytes, body: bytes) -> bytes: return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body)) @@ -359,6 +367,7 @@ class Frontend: self.video = hashlib.sha1() self.audio = hashlib.sha1() self.frame_hashes: list[list] = [] + self.uniform: set[str] = set() self.logs: list[dict] = [] self.presses: list[tuple[int, int, int]] = [tuple(p) for p in spec["presses"]] self.unanswered: dict[int, int] = {} @@ -433,7 +442,10 @@ class Frontend: self.video.update(struct.pack(" dict: observables[f"state:{name}"] = state_vars.get(name) if spec["frame_hash_every"]: observables["frame_hashes"] = front.frame_hashes + observables["uniform_frames"] = sorted(front.uniform) core.retro_unload_game() core.retro_deinit() return { @@ -584,7 +597,7 @@ def frontend_identity() -> dict: def core_identity(args: argparse.Namespace) -> dict: core = Path(args.core) - return { + identity = { "file": core.name, "sha256": file_digest(core, "sha256"), "repo": args.core_repo, @@ -592,6 +605,9 @@ def core_identity(args: argparse.Namespace) -> dict: "build_command": args.build_command, "compiler": elf_comment(core) or None, } + if args.core_artifact: + identity["artifact"] = args.core_artifact + return identity # -------------------------------------------------------------------------- @@ -769,6 +785,8 @@ def build_parser() -> argparse.ArgumentParser: common.add_argument("--core-repo", help="source repository URL of the core build") common.add_argument("--core-commit", help="commit the core was built from") common.add_argument("--build-command", help="command and flags that built the core") + common.add_argument("--core-artifact", metavar="URL", + help="where the .so was downloaded from, such as a buildbot nightly") common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT), help="append the record as one line of measurements/.jsonl " "in the checkout DIR (default: this one)") @@ -841,7 +859,7 @@ def validate_args(args: argparse.Namespace) -> list[str]: problems.append(f"--vary {dest}={src}: not a file") names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var} if args.frame_hash_every: - names.add("frame_hashes") + names.update(("frame_hashes", "uniform_frames")) if args.observable not in names: problems.append(f"--observable {args.observable}: not one of {', '.join(sorted(names))}") return problems @@ -936,8 +954,9 @@ def default_subject(record: dict) -> str: return f"{record['kind']}-{slugify(content, 30)}-{digest}" -def write_record(directory: Path, full: dict) -> Path: - """Append the record, with its id, to its emulator's .jsonl.""" +def write_record(directory: Path, full: dict) -> tuple[Path, dict]: + """Append the record, with its id, to its emulator's .jsonl. Returns the + file and the record as written.""" from measurement_records import line_text, parse_lines, records_path, seal # noqa: PLC0415 sealed = seal(full) @@ -948,7 +967,7 @@ def write_record(directory: Path, full: dict) -> Path: lines.parent.mkdir(parents=True, exist_ok=True) with lines.open("a", encoding="utf-8") as handle: handle.write(line_text(sealed) + "\n") - return lines + return lines, sealed def probe(args: argparse.Namespace) -> int: @@ -999,7 +1018,7 @@ def probe(args: argparse.Namespace) -> int: "repetitions": repetitions, "presses": [{"frame": f, "button": b, "length": n} for f, b, n in args.press], "observables": list(BASE_OBSERVABLES) + [f"state:{v}" for v in args.state_var] - + (["frame_hashes"] if args.frame_hash_every else []), + + (["frame_hashes", "uniform_frames"] if args.frame_hash_every else []), "frame_hash_every": args.frame_hash_every, "joypad_ports": args.joypad_ports, }, @@ -1025,7 +1044,7 @@ def probe(args: argparse.Namespace) -> int: record["subject"] = args.subject or default_subject(record) if args.record: try: - lines = write_record(Path(args.record), record) + lines, record = write_record(Path(args.record), record) except ProbeError as exc: print(f"ERROR: {exc}", file=sys.stderr) return 2 diff --git a/scripts/measurement_records.py b/scripts/measurement_records.py index ce7e685f1..d23db29b0 100644 --- a/scripts/measurement_records.py +++ b/scripts/measurement_records.py @@ -81,6 +81,47 @@ def records_path(emulator: str) -> str: return f"{RECORDS_DIR}/{emulator}.jsonl" +# The campaign's own files beside the records: append-only like them, one +# line per finding or per pass, each line hashed to its id. No profile takes +# these names (tests/test_campaign.py checks it). +FINDINGS = "findings" +CAMPAIGN_RUNS = "campaign-runs" +CAMPAIGN_FILES = { + FINDINGS: ("found_at", "campaign-finding.schema.json"), + CAMPAIGN_RUNS: ("started_at", "campaign-run.schema.json"), +} + + +def line_id(line: dict, date_key: str) -> str: + body = {key: value for key, value in line.items() if key != "id"} + return f"{line[date_key][:10]}-{sha256_hex(canonical(body))[:12]}" + + +def seal_line(line: dict, date_key: str) -> dict: + return {**line, "id": line_id(line, date_key)} + + +def read_campaign(root: Path, name: str) -> list[dict]: + """The lines of one campaign file, each checked against its id.""" + path = root / records_path(name) + if not path.is_file(): + return [] + date_key = CAMPAIGN_FILES[name][0] + lines = parse_lines(path.read_text(encoding="utf-8"), records_path(name)) + problems = [] + seen: set[str] = set() + for number, line in enumerate(lines, 1): + rid = str(line.get("id", "")) + if not ID_PATTERN.match(rid) or date_key not in line or line_id(line, date_key) != rid: + problems.append(f"{records_path(name)}:{number}: id {rid!r} does not hash the line") + if rid in seen: + problems.append(f"{records_path(name)}:{number}: id {rid} appears twice") + seen.add(rid) + if problems: + raise MeasurementsError("\n ".join(problems)) + return lines + + IDENTITY = ("id", "emulator", "subject", "supersedes", "migrated_from_version") @@ -177,6 +218,8 @@ def read_records(root: Path) -> list[Stored]: if not path.endswith(".jsonl") or "/" in emulator: problems.append(f"{path}: {RECORDS_DIR}/ holds .jsonl files only") continue + if emulator in CAMPAIGN_FILES: + continue seen: set[str] = set() try: records = parse_lines(files[path].read_text(encoding="utf-8"), path) diff --git a/scripts/sitemeasurements.py b/scripts/sitemeasurements.py index 8d6702727..146b778aa 100644 --- a/scripts/sitemeasurements.py +++ b/scripts/sitemeasurements.py @@ -140,6 +140,80 @@ def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict, } +def campaign_summary(root: Path, stored: list[mr.Stored], source: dict | None) -> dict: + """The last campaign pass and the findings still waiting for review. + + A finding's history is its lines under one key: the latest line is its + state, and only an open or flagged one is listed. + """ + runs = mr.read_campaign(root, mr.CAMPAIGN_RUNS) + latest: dict[str, dict] = {} + applied = 0 + for line in mr.read_campaign(root, mr.FINDINGS): + latest[line["key"]] = line + applied += line["status"] == "applied" + where = {s.record["id"]: s for s in stored} + links = {} + for line in latest.values(): + for rid in line["evidence"]: + found = where.get(rid) + if found and source and source.get("commit"): + links[rid] = permalink(source["repo"], source["commit"], + mr.records_path(found.emulator), found.line) + open_items = [{**line, "links": [links.get(rid) for rid in line["evidence"]]} + for _, line in sorted(latest.items()) if line["status"] in ("open", "flagged")] + return {"last_run": runs[-1] if runs else None, "passes": len(runs), "applied": applied, + "open": open_items} + + +def _value(value: object) -> str: + return f"`{json.dumps(value)}`" if value is not None else "nothing" + + +def campaign_section(summary: dict | None) -> list[str]: + """The campaign's last pass and its open findings, for the explorer page.""" + if not summary or not summary.get("last_run"): + return [] + last = summary["last_run"] + counts = last["counts"] + lines = [ + "## Campaign", + "", + f"Tests run by themselves every night on an idle computer ([how]({METHOD_PAGE})). " + f"Last pass: {last['ended_at'][:10]}, {_text(last['result'])}" + + (f" ({_text(last['detail'])})" if last.get("detail") else "") + ". " + f"{counts['done']} tests done, {counts['failed']} failed, {counts['pending']} waiting. " + f"{summary['applied']} profile fields were set from a run.", + "", + ] + if last["pending_reasons"]: + lines += ["| Waiting for | Tests |", "|-------------|------:|"] + lines += [f"| {_text(reason)} | {number} |" + for reason, number in sorted(last["pending_reasons"].items(), key=lambda kv: (-kv[1], kv[0]))] + lines.append("") + if last["failed"]: + lines += ["Failed tests:", ""] + lines += [f"- {_text(item['emulator'])}, `{_text(item['check'])}`: {_text(item['reason'])}" + for item in last["failed"]] + lines.append("") + lines += ["### Open findings", ""] + if not summary["open"]: + lines += ["None: every finding was settled by a rule or resolved by a later run.", ""] + return lines + lines += ["A finding is what a run showed that no rule settles. It waits for someone to " + "read the emulator's code.", ""] + for item in summary["open"]: + records = ", ".join(f"[{rid}]({link})" if link else f"`{rid}`" + for rid, link in zip(item["evidence"], item["links"])) + lines += [ + f"- **{_text(item['emulator'])}**, `{_text(item['file'])}`, `{_text(item['field'])}` " + f"({item['status']}): the profile says {_value(item['profile_says'])}, the run shows " + f"{_value(item['run_shows'])}. {_text(item['statement'])}. Records: {records}.", + "", + ] + return lines + + # -------------------------------------------------------------------------- # SQLite # -------------------------------------------------------------------------- @@ -360,6 +434,7 @@ def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulato source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)} profiles = load_profiles(emulators_dir) index = build_index(stored, source, profiles, system_names) + index["campaign"] = campaign_summary(root, stored, source) paths = [public / INDEX_PATH] (public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True) write_if_changed(str(public / INDEX_PATH), @@ -485,6 +560,7 @@ def generate_explorer_page(index: dict) -> str: "", "", "", + *campaign_section(index.get("campaign")), ]) diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index a870d4736..b00141ea4 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -601,6 +601,8 @@ def _measurement_errors(root: Path | None = None) -> list[str]: root = root or ROOT validator = _validator("core-probe-record.schema.json") out: list[str] = [] + campaign = {mr.records_path(name): _validator(schema) + for name, (_, schema) in mr.CAMPAIGN_FILES.items()} for path, file in sorted(mr.tree_files(root).items()): if not path.endswith(".jsonl"): continue @@ -610,10 +612,12 @@ def _measurement_errors(root: Path | None = None) -> list[str]: except json.JSONDecodeError as exc: out.append(f"{path}:{number}: {exc}") continue - out.extend(_errors(validator, record, f"{path}:{number}")) + out.extend(_errors(campaign.get(path, validator), record, f"{path}:{number}")) if not out: try: mr.read_records(root) + for name in mr.CAMPAIGN_FILES: + mr.read_campaign(root, name) except mr.MeasurementsError as exc: out.append(str(exc)) return out diff --git a/tests/campaign/content.yml b/tests/campaign/content.yml new file mode 100644 index 000000000..512739d5f --- /dev/null +++ b/tests/campaign/content.yml @@ -0,0 +1,42 @@ +# Test content of the real-condition campaign (scripts/campaign.py), per system. +# +# Each entry names a catalog entry, never a file: `name` is the exact Redump or +# No-Intro name romfetch fetches and verifies (`romfetch get `), +# and `reason` says why this game serves the test. Content is read from +# .cache/test-content/// with its identity.json, and never +# enters git. A record keeps only its identity (catalog name, sha1, tracks). +# +# region the territory of the disc or cartridge: the BIOS entry the +# boot test varies is the one whose `region:` holds it, or one +# without `region:` +# load the file handed to the core: `cue` (the .cue sheet) or a +# file name inside the folder; by default the single file +# frames frames run per repetition, the same with and without the file +# frame_hash_every frames between two sampled frame hashes +# pinned never deleted by the campaign, whoever fetched it +# boot_file (per system) the declared file the boot test varies, when a +# profile declares it under this name or an alias +# +# The checks a game is used for are the boot test (no file against the +# declared file), the files test (every declared file staged, which ones the +# core opens) and the benchmark. + +systems: + sony-playstation: + romfetch: psx + # The file the boot test varies, when the profile declares it: the US + # BIOS of a US disc, the same file for every core so the runs compare. + boot_file: scph5501.bin + content: + - name: "Bust-A-Move 2 - Arcade Edition (USA)" + catalog: redump + reason: >- + A US disc that goes through the BIOS shell and its licence screen, then + reaches the publisher logos and the attract mode within a minute, with + no button pressed. Several static screens, so a run with and a run + without the BIOS can be compared frame by frame. + region: north-america + load: cue + frames: 3600 + frame_hash_every: 10 + pinned: true diff --git a/tests/test_campaign.py b/tests/test_campaign.py new file mode 100644 index 000000000..bb721b5a6 --- /dev/null +++ b/tests/test_campaign.py @@ -0,0 +1,756 @@ +"""campaign.py runs the real-condition campaign: state, guards, content, rules. + +No network and no real core. The rules and the profile editor are checked on +synthetic records and profiles, the guards and the probe retries on a fake +host, the content store on folders made in the test. One class builds a +libretro core from C, when a compiler is there, and runs whole passes on a +synthetic checkout: records, rules applied with their evidence, findings, +resume and rerun on a changed core. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import MappingProxyType + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import campaign +import campaign_profiles +import campaign_rules as rules +import measurement_records as mr +import sitemeasurements + +TMP = REPO_ROOT / "tmp" / "tests" + +PROFILE = """\ +emulator: Fake +type: libretro +cores: [fake] +systems: [fake-system] +notes: > + A profile for the tests. + +files: + # -- the BIOS -- + + - name: "bios.bin" + description: "The BIOS" + required: true + size: 4 + validation: [size] + aliases: + - "Bios.bin" + + - name: "other.bin" + required: false +""" + +FAKE_CORE = r""" +#include +#include +#include +#include +#include +#include +struct retro_system_info { const char *n, *v, *e; bool full, block; }; +struct retro_game_geometry { unsigned bw, bh, mw, mh; float a; }; +struct retro_system_timing { double fps, rate; }; +struct retro_system_av_info { struct retro_game_geometry g; struct retro_system_timing t; }; +typedef bool (*env_t)(unsigned, void *); +typedef void (*video_t)(const void *, unsigned, unsigned, size_t); +typedef size_t (*batch_t)(const int16_t *, size_t); +typedef int16_t (*input_t)(unsigned, unsigned, unsigned, unsigned); +static env_t env; static video_t video; static batch_t batch; +static uint16_t pixels[4 * 2]; static int booted; static unsigned frame; +void retro_set_environment(env_t e) { bool t = true; env = e; env(18, &t); } +void retro_set_video_refresh(video_t v) { video = v; } +void retro_set_audio_sample(void *f) { (void)f; } +void retro_set_audio_sample_batch(batch_t b) { batch = b; } +void retro_set_input_poll(void *f) { (void)f; } +void retro_set_input_state(input_t f) { (void)f; } +void retro_set_controller_port_device(unsigned port, unsigned device) { (void)port; (void)device; } +void retro_get_system_info(struct retro_system_info *i) { + i->n = "Fake"; i->v = "1"; i->e = ""; i->full = true; i->block = false; } +void retro_get_system_av_info(struct retro_system_av_info *i) { + memset(i, 0, sizeof *i); i->t.fps = 60.0; i->t.rate = 44100.0; } +void retro_init(void) {} +void retro_deinit(void) {} +static FILE *try_open(const char *dir, const char *name) { + char path[4096]; snprintf(path, sizeof path, "%s/%s", dir, name); return fopen(path, "rb"); } +bool retro_load_game(const void *g) { + const char *dir = NULL; FILE *f; (void)g; env(9, &dir); + f = try_open(dir, "bios.bin"); + if (!f) f = try_open(dir, "BIOS.BIN"); + if (f) { fclose(f); booted = 1; } else booted = getenv("FAKE_HLE") != NULL; + return true; } +void retro_unload_game(void) {} +void retro_run(void) { + int16_t s[4] = {1, 2, 3, 4}; + if (getenv("FAKE_HANG")) for (;;) {} + for (int i = 0; i < 8; i++) pixels[i] = booted ? (uint16_t)(frame / 2 + i) : 0; + frame++; + video(pixels, 4, 2, 8); batch(s, 2); } +size_t retro_serialize_size(void) { return 0; } +bool retro_serialize(void *d, size_t n) { (void)d; (void)n; return false; } +""" + + +def rep(frames: list[str], uniform: list[str] | None = None, loaded: bool = True) -> dict: + return {"loaded": loaded, "observables": { + "frame_hashes": [[i * 10, h] for i, h in enumerate(frames)], + "uniform_frames": sorted(set(uniform or []))}} + + +def boot_record(absent: list[dict], present: list[dict], rid: str = "2026-10-27-aaaaaaaaaaaa") -> dict: + return {"id": rid, "variants": [{"label": "bios.bin-absent", "repetitions": absent}, + {"label": "bios.bin", "repetitions": present}]} + + +class BootOutcome(unittest.TestCase): + GAME = ("black", "logo", "title", "title2") + + def test_boots_when_screens_are_shared(self) -> None: + record = boot_record([rep(["black", "hle", "logo", "title"], ["black"])] * 2, + [rep(self.GAME, ["black"])] * 2) + boot = rules.boot_outcome(record, "bios.bin-absent", "bios.bin") + self.assertEqual((boot.status, boot.shared), (rules.BOOTS, 2)) + + def test_one_colour_and_no_load_fail(self) -> None: + black = boot_record([rep(["black"], ["black"]), rep([], [], loaded=False)], + [rep(self.GAME, ["black"])]) + self.assertEqual(rules.boot_outcome(black, "bios.bin-absent", "bios.bin").status, rules.FAILS) + + def test_a_shared_blank_screen_proves_nothing(self) -> None: + # The black frame both runs start on is one colour: it is no game screen. + record = boot_record([rep(["black", "error"], ["black"])], [rep(self.GAME, ["black"])]) + self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status, + rules.INCONCLUSIVE) + + def test_one_shared_screen_is_not_enough(self) -> None: + record = boot_record([rep(["logo", "error"])], [rep(self.GAME, ["black"])]) + self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status, + rules.INCONCLUSIVE) + + def test_repetitions_that_disagree_settle_nothing(self) -> None: + record = boot_record([rep(["logo", "title"]), rep(["black"], ["black"])], + [rep(self.GAME, ["black"])]) + self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status, + rules.INCONCLUSIVE) + + def test_a_reference_that_shows_nothing_settles_nothing(self) -> None: + record = boot_record([rep(["logo", "title"])], [rep(["black"], ["black"])]) + self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status, + rules.INCONCLUSIVE) + record = boot_record([rep(["logo", "title"])], [rep(self.GAME, loaded=False)]) + self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status, + rules.INCONCLUSIVE) + + +class RuleTable(unittest.TestCase): + ENTRY = MappingProxyType({"name": "bios.bin", "required": True, "aliases": ["Bios.bin"]}) + RID = "2026-10-27-bbbbbbbbbbbb" + + def test_boots_without_settles_hle_and_queues_required(self) -> None: + verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "x", 3), self.ENTRY, self.RID) + self.assertEqual([(v.field, v.value, v.apply) for v in verdicts], + [("hle_fallback", True, True), ("required", False, False)]) + self.assertEqual(verdicts[0].evidence, (self.RID,)) + optional = rules.rule_bios_need(rules.Boot(rules.BOOTS, "x", 3), + {"name": "a", "required": False}, self.RID) + self.assertEqual([v.field for v in optional], ["hle_fallback"]) + + def test_fails_without_settles_required_and_no_hle(self) -> None: + verdicts = rules.rule_bios_need(rules.Boot(rules.FAILS, "x"), self.ENTRY, self.RID) + self.assertEqual([(v.field, v.value, v.apply) for v in verdicts], + [("required", True, True), ("hle_fallback", False, True)]) + + def test_inconclusive_is_queued(self) -> None: + (verdict,) = rules.rule_bios_need(rules.Boot(rules.INCONCLUSIVE, "x"), self.ENTRY, self.RID) + self.assertFalse(verdict.apply) + + def test_path_and_alias(self) -> None: + self.assertIsNone(rules.rule_path(self.ENTRY, "bios.bin", rules.Opened("bios.bin", self.RID))) + moved = rules.rule_path(self.ENTRY, "bios.bin", rules.Opened("sys/bios.bin", self.RID)) + self.assertEqual((moved.field, moved.value, moved.apply), ("path", "sys/bios.bin", True)) + self.assertIsNone(rules.rule_alias(self.ENTRY, rules.Opened("Bios.bin", self.RID))) + alias = rules.rule_alias(self.ENTRY, rules.Opened("BIOS.BIN", self.RID)) + self.assertEqual((alias.field, alias.value), ("aliases", ["Bios.bin", "BIOS.BIN"])) + + def test_never_opened_is_flagged_not_applied(self) -> None: + verdict = rules.rule_never_opened(self.ENTRY, [self.RID], "Game (USA)") + self.assertFalse(verdict.apply) + self.assertEqual(verdict.rule, "never_opened") + + def test_variants(self) -> None: + self.assertIsNone(rules.rule_variants("same", ["a", "b"], self.ENTRY, self.RID)) + self.assertFalse(rules.rule_variants("differs", ["b", "a"], self.ENTRY, self.RID).apply) + + def test_agreement_and_evidence(self) -> None: + self.assertTrue(rules.agrees({"name": "a"}, rules.Verdict("r", "hle_fallback", False, "", ()))) + self.assertFalse(rules.agrees({"name": "a"}, rules.Verdict("r", "hle_fallback", True, "", ()))) + self.assertTrue(rules.agrees(self.ENTRY, rules.Verdict("r", "aliases", ["Bios.bin"], "", ()))) + merged = rules.merge_evidence({"required": ["x"]}, rules.Verdict("r", "required", True, "", ("y", "x"))) + self.assertEqual(merged, {"required": ["x", "y"]}) + + +class ProfileEditor(unittest.TestCase): + def test_replace_insert_and_keep_the_rest(self) -> None: + out = campaign_profiles.set_fields(PROFILE, 0, { + "required": False, "hle_fallback": True, + "aliases": ["Bios.bin", "BIOS.BIN"], "evidence": {"hle_fallback": ["2026-10-27-cccccccccccc"]}}) + parsed = yaml.safe_load(out) + self.assertEqual(parsed["files"][0]["hle_fallback"], True) + self.assertEqual(parsed["files"][0]["aliases"], ["Bios.bin", "BIOS.BIN"]) + self.assertEqual(parsed["files"][1], {"name": "other.bin", "required": False}) + self.assertIn(" # -- the BIOS --", out) + self.assertIn(' required: false\n hle_fallback: true\n', out) + self.assertIn(' - "BIOS.BIN"\n', out) + lines = out.splitlines() + self.assertEqual(lines[lines.index(' evidence:') + 1], ' hle_fallback: ["2026-10-27-cccccccccccc"]') + + def test_the_last_entry_and_a_no_op(self) -> None: + out = campaign_profiles.set_fields(PROFILE, 1, {"path": "sub/other.bin"}) + self.assertEqual(yaml.safe_load(out)["files"][1]["path"], "sub/other.bin") + self.assertEqual(campaign_profiles.set_fields(PROFILE, 1, {"required": False}), PROFILE) + + def test_refusals(self) -> None: + with self.assertRaises(campaign_profiles.ProfileEditError): + campaign_profiles.set_fields(PROFILE, 2, {"required": False}) + with self.assertRaises(campaign_profiles.ProfileEditError): + campaign_profiles.set_fields("emulator: X\n", 0, {"required": False}) + with self.assertRaises(campaign_profiles.ProfileEditError): + campaign_profiles.set_fields(PROFILE, 0, {"name": "x.bin"}) + + def test_every_profile_reads_as_its_entries(self) -> None: + # The text layout the editor reads holds in every profile it may edit. + for path in sorted((REPO_ROOT / "emulators").glob("*.yml")): + text = path.read_text(encoding="utf-8") + entries = (yaml.safe_load(text) or {}).get("files") or [] + if not entries or "\nfiles:\n" not in text: + continue + with self.subTest(profile=path.name): + self.assertEqual(len(campaign_profiles.item_spans(text.splitlines())), len(entries)) + + +class FakeHost(campaign.Host): + def __init__(self, loads: list[float] | None = None, free: float = 500.0) -> None: + super().__init__() + self.loads = list(loads or [0.0]) + self.now = 0.0 + self.slept: list[float] = [] + self.free = free + self.calls: list[list[str]] = [] + self.results: list[object] = [] + self.loadavg = self._load + self.sleep = self._sleep + self.clock = lambda: self.now + self.free_gb = lambda path: self.free + self.cpu_count = 4 + self.run = self._run + + def _load(self) -> tuple[float, float, float]: + value = self.loads.pop(0) if len(self.loads) > 1 else self.loads[0] + return (value, value, value) + + def _sleep(self, seconds: float) -> None: + self.slept.append(seconds) + self.now += seconds + + def _run(self, argv, **kwargs): + self.calls.append(list(argv)) + result = self.results.pop(0) if len(self.results) > 1 else self.results[0] + if isinstance(result, BaseException): + raise result + return result + + +class Workspace(unittest.TestCase): + def setUp(self) -> None: + TMP.mkdir(parents=True, exist_ok=True) + self.tmp = Path(tempfile.mkdtemp(dir=TMP)) + self.root = self.tmp / "repo" + (self.root / "emulators").mkdir(parents=True) + (self.root / "emulators" / "fake.yml").write_text(PROFILE, encoding="utf-8") + bios = self.root / "bios" / "Fake" / "bios.bin" + bios.parent.mkdir(parents=True) + bios.write_bytes(b"BIOS") + sha1 = hashlib.sha1(b"BIOS").hexdigest() + db = {"files": {sha1: {"path": "bios/Fake/bios.bin", "name": "bios.bin", "size": 4, + "sha1": sha1, "md5": hashlib.md5(b"BIOS").hexdigest()}}, + "indexes": {"by_name": {"bios.bin": [sha1]}, "by_md5": {}, "by_crc32": {}, + "by_sha256": {}, "by_path_suffix": {}}} + (self.root / "database.json").write_text(json.dumps(db), encoding="utf-8") + self.content_file = self.tmp / "content.yml" + self.content_file.write_text(yaml.safe_dump({"systems": {"fake-system": { + "romfetch": "fakesys", "content": [{ + "name": "Fake Game (World)", "catalog": "no-intro", "reason": "a test", + "frames": 12, "frame_hash_every": 2}]}}}), encoding="utf-8") + self.game = self.tmp / "content" / "fakesys" / "Fake Game (World)" + self.paths = campaign.Paths(root=self.root, home=self.tmp / "home", + content=self.tmp / "content", db=self.root / "database.json", + content_file=self.content_file) + + def tearDown(self) -> None: + shutil.rmtree(self.tmp, ignore_errors=True) + + def put_game(self, folder: Path | None = None, payload: bytes = b"GAME") -> Path: + folder = folder or self.game + folder.mkdir(parents=True, exist_ok=True) + (folder / "game.bin").write_bytes(payload) + (folder / "identity.json").write_text(json.dumps({"status": "verified", "files": [ + {"name": "game.bin", "size": len(payload), "sha1": hashlib.sha1(payload).hexdigest()}]})) + return folder + + def make(self, host: campaign.Host | None = None, **settings: object) -> campaign.Campaign: + values = {"delays": (1, 2), "echo": False, **settings} + return campaign.Campaign(self.paths, campaign.Settings(**values), host or FakeHost(), + lock_path=self.tmp / "auto.lock") + + +class StateAndResume(Workspace): + def test_should_run(self) -> None: + self.assertTrue(campaign.should_run(None, "a", "d")) + done = {"status": "done", "core_sha256": "a", "profile_digest": "d"} + self.assertFalse(campaign.should_run(done, "a", "d")) + self.assertTrue(campaign.should_run(done, "b", "d")) + self.assertTrue(campaign.should_run(done, "a", "e")) + self.assertTrue(campaign.should_run({**done, "status": "pending"}, "a", "d")) + self.assertFalse(campaign.should_run({**done, "status": "failed"}, "a", "d")) + + def test_evidence_does_not_change_the_digest(self) -> None: + profile = yaml.safe_load(PROFILE) + before = campaign.profile_digest(profile) + profile["files"][0]["evidence"] = {"required": ["2026-10-27-dddddddddddd"]} + self.assertEqual(campaign.profile_digest(profile), before) + profile["files"][0]["required"] = False + self.assertNotEqual(campaign.profile_digest(profile), before) + + def test_state_survives_a_reload(self) -> None: + c = self.make() + c.pending("fake", "fake/fake-system/x/files", "content: romfetch not found") + campaign.save_state(self.paths.state, c.state) + again = self.make() + self.assertEqual(again.state["checks"]["fake"]["fake/fake-system/x/files"]["status"], "pending") + + def test_budget(self) -> None: + self.assertEqual(campaign.parse_budget("6h"), 21600) + self.assertEqual(campaign.parse_budget("90m"), 5400) + self.assertEqual(campaign.parse_budget("30"), 30) + + +class Guards(Workspace): + def test_busy_host_waits_then_runs(self) -> None: + host = FakeHost(loads=[5.0, 5.0, 0.5]) + c = self.make(host, max_load_wait=600) + c.deadline = 10_000 + c.guard() + self.assertEqual(host.slept, [60.0, 60.0]) + + def test_busy_host_pauses(self) -> None: + host = FakeHost(loads=[5.0]) + c = self.make(host, max_load_wait=120) + c.deadline = 10_000 + with self.assertRaisesRegex(campaign.Pause, "busy host"): + c.guard() + + def test_full_disk_pauses(self) -> None: + c = self.make(FakeHost(free=10.0), min_free_gb=30) + c.deadline = 10_000 + with self.assertRaisesRegex(campaign.Pause, "disk"): + c.guard() + + def test_budget_pauses(self) -> None: + c = self.make() + c.deadline = 0 + with self.assertRaisesRegex(campaign.Pause, "budget"): + c.guard() + + def test_the_lock_is_skipped_under_autorun(self) -> None: + lock = self.tmp / "held.lock" + with campaign.autorun.global_lock(lock): + with self.assertRaises(campaign.autorun.LockBusy), campaign.host_lock(lock): + pass + os.environ[campaign.LOCK_HELD_ENV] = "1" + try: + with campaign.host_lock(lock): + pass + finally: + del os.environ[campaign.LOCK_HELD_ENV] + + +class ProbeRuns(Workspace): + def completed(self, code: int, stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess: + return subprocess.CompletedProcess([], code, stdout, stderr) + + def probe(self, *results: object) -> tuple[campaign.ProbeResult, FakeHost]: + host = FakeHost() + host.results = list(results) + c = self.make(host) + c.deadline = 10_000 + return c.probe(["run", "--core", "x.so"], "fake-run"), host + + def test_timeout_is_failed_and_not_retried(self) -> None: + result, host = self.probe(subprocess.TimeoutExpired(["x"], 1)) + self.assertEqual((result.status, result.reason), ("failed", "timeout")) + self.assertEqual(len(host.calls), 1) + self.assertIn("killed after", Path(result.log).read_text()) + result, host = self.probe(self.completed(1, stderr="subprocess.TimeoutExpired: timed out after 900 seconds")) + self.assertEqual((result.reason, len(host.calls)), ("timeout", 1)) + + def test_a_crashing_core_is_tried_three_times(self) -> None: + result, host = self.probe(self.completed(1, stderr="worker exited -11")) + self.assertEqual((result.status, result.reason), ("failed", "crashed 3 times")) + self.assertEqual(len(host.calls), 3) + + def test_a_crash_then_a_record(self) -> None: + record = {"id": "2026-10-27-eeeeeeeeeeee"} + result, host = self.probe(self.completed(1, stderr="worker exited -11"), + self.completed(0, stdout=json.dumps(record))) + self.assertEqual((result.status, result.record, len(host.calls)), ("done", record, 2)) + + def test_refusal_on_a_busy_host_pauses(self) -> None: + with self.assertRaisesRegex(campaign.Pause, "busy host"): + self.probe(self.completed(3, stderr="load average")) + + +class Content(Workspace): + def content(self, name: str = "Fake Game (World)", pinned: bool = False) -> campaign.Content: + return campaign.Content("fake-system", "fakesys", name, "no-intro", "r", None, "", 12, 2, + pinned, None) + + def test_local_content_is_verified_by_its_identity(self) -> None: + self.put_game() + c = self.make() + self.assertEqual(c.store.get(self.content()), self.game) + (self.game / "game.bin").write_bytes(b"EDIT") + with self.assertRaisesRegex(campaign.Unavailable, "does not match"): + self.make().store.get(self.content()) + + def test_without_romfetch_the_check_waits_for_content(self) -> None: + c = self.make(romfetch=str(self.tmp / "absent-romfetch")) + with self.assertRaisesRegex(campaign.Unavailable, "romfetch not found"): + c.store.get(self.content()) + + def test_romfetch_is_retried_on_a_source_error(self) -> None: + fetcher = self.tmp / "romfetch" + fetcher.write_text("#!/bin/sh\n") + fetcher.chmod(0o755) + host = FakeHost() + error = json.dumps({"error": {"code": "network", "message": "m", "hint": "h"}}) + game = self.game + + def fetched(*args, **kwargs): + self.put_game(game) + return subprocess.CompletedProcess([], 0, '{"status": "verified"}', "") + host.results = [subprocess.CompletedProcess([], 3, error, ""), None] + host.run = lambda argv, **kw: (host.calls.append(argv), fetched() if len(host.calls) == 2 + else host.results[0])[1] + c = self.make(host, romfetch=str(fetcher)) + self.assertEqual(c.store.get(self.content()), self.game) + self.assertEqual(host.slept, [1]) + self.assertEqual(host.calls[0][1:4], ["get", "fakesys", "Fake Game (World)"]) + self.assertTrue(c.state["content"]["fakesys/Fake Game (World)"]["fetched_by_campaign"]) + + def test_romfetch_not_found_names_its_code(self) -> None: + fetcher = self.tmp / "romfetch" + fetcher.write_text("#!/bin/sh\n") + host = FakeHost() + host.results = [subprocess.CompletedProcess( + [], 1, json.dumps({"error": {"code": "no_entry", "message": "m", "hint": "h"}}), "")] + c = self.make(host, romfetch=str(fetcher)) + with self.assertRaisesRegex(campaign.Unavailable, r"exit 1 \(no_entry\)"): + c.store.get(self.content()) + self.assertEqual(len(host.calls), 1) + + def test_shared_content_stays_until_its_last_user(self) -> None: + c = self.make() + self.put_game() + content = self.content() + c.state["content"][content.key] = {"fetched_by_campaign": True, "pinned": False, "size": 4} + c.store.hold(content, "alpha") + c.store.hold(content, "beta") + c.store.release(content, "alpha") + self.assertTrue(self.game.is_dir()) + c.store.release(content, "beta") + self.assertFalse(self.game.exists()) + + def test_pinned_or_foreign_content_is_never_deleted(self) -> None: + c = self.make() + self.put_game() + content = self.content(pinned=True) + c.state["content"][content.key] = {"fetched_by_campaign": True, "pinned": True} + c.store.hold(content, "alpha") + c.store.release(content, "alpha") + self.assertTrue(self.game.is_dir()) + c.state["content"][content.key] = {"fetched_by_campaign": False, "pinned": False} + c.store.hold(content, "alpha") + c.store.release(content, "alpha") + self.assertTrue(self.game.is_dir()) + + def test_least_recently_used_goes_first_past_the_cap(self) -> None: + c = self.make(cache_cap_gb=5e-9) + for name, used in (("Old", "2026-01-01"), ("New", "2026-06-01"), ("Kept", "2025-01-01")): + self.put_game(self.tmp / "content" / "fakesys" / name) + c.state["content"][f"fakesys/{name}"] = {"fetched_by_campaign": True, "pinned": name == "Kept", + "size": 4, "last_used": used} + self.assertEqual(c.store.evict(keep=set()), ["fakesys/Old"]) + self.assertTrue((self.tmp / "content" / "fakesys" / "New").is_dir()) + self.assertTrue((self.tmp / "content" / "fakesys" / "Kept").is_dir()) + + def test_load_file(self) -> None: + self.put_game() + self.assertEqual(campaign.load_file(self.game, ""), self.game / "game.bin") + with self.assertRaises(campaign.Unavailable): + campaign.load_file(self.game, "cue") + + +class Findings(Workspace): + RID = "2026-10-27-ffffffffffff" + + def setUp(self) -> None: + super().setUp() + self.c = self.make() + self.c.current = campaign.Pass(started=campaign.now_iso()) + self.content = campaign.Content("fake-system", "fakesys", "Fake Game (World)", "no-intro", + "r", None, "", 12, 2, False, None) + self.core = campaign.Core("fake", self.tmp / "fake_libretro.so", "a" * 64) + self.profile = self.root / "emulators" / "fake.yml" + + def lines(self) -> list[dict]: + return mr.read_campaign(self.root, mr.FINDINGS) + + def entry(self, index: int = 0) -> dict: + return yaml.safe_load(self.profile.read_text())["files"][index] + + def test_a_settled_field_is_written_with_its_evidence(self) -> None: + verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID) + self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts) + entry = self.entry() + self.assertEqual(entry["hle_fallback"], True) + self.assertEqual(entry["evidence"], {"hle_fallback": [self.RID]}) + self.assertEqual(entry["required"], True) + self.assertEqual([(line["field"], line["status"]) for line in self.lines()], + [("hle_fallback", "applied"), ("required", "open")]) + self.assertIn("emulators/fake.yml", self.c.current.touched) + # The same outcome a second time adds nothing. + self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts) + self.assertEqual(len(self.lines()), 2) + + def test_an_agreeing_run_adds_evidence_only(self) -> None: + verdicts = rules.rule_bios_need(rules.Boot(rules.FAILS, "nothing"), self.entry(), self.RID) + self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts) + entry = self.entry() + self.assertEqual((entry["required"], entry["hle_fallback"]), (True, False)) + self.assertEqual(entry["evidence"], {"hle_fallback": [self.RID], "required": [self.RID]}) + self.assertEqual(self.lines(), []) + + def test_a_later_agreement_resolves_the_open_finding(self) -> None: + (verdict,) = rules.rule_bios_need(rules.Boot(rules.INCONCLUSIVE, "?"), self.entry(), self.RID) + self.c.settle("fake", self.content, self.core, 0, self.entry(), [verdict]) + self.assertEqual(self.lines()[-1]["status"], "open") + later = rules.Verdict("inconclusive", "hle_fallback", False, "agrees now", ("2026-10-28-000000000000",)) + self.c.settle("fake", self.content, self.core, 0, self.entry(), [later]) + self.assertEqual([line["status"] for line in self.lines()], ["open", "resolved"]) + + def test_never_opened_is_flagged(self) -> None: + verdict = rules.rule_never_opened(self.entry(1), [self.RID], "Fake Game (World)") + self.c.settle("fake", self.content, self.core, 1, self.entry(1), [verdict]) + self.assertEqual(self.lines()[0]["status"], "flagged") + self.assertNotIn("evidence", self.entry(1)) + + def test_finding_lines_validate(self) -> None: + verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID) + self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts) + self.c.write_run_line(self.c.current) + try: + from jsonschema import Draft202012Validator + except ImportError: + self.skipTest("jsonschema is a tooling dependency") + for name, (_, schema_name) in mr.CAMPAIGN_FILES.items(): + schema = json.loads((REPO_ROOT / "schemas" / schema_name).read_text()) + for line in mr.read_campaign(self.root, name): + self.assertEqual(list(Draft202012Validator(schema).iter_errors(line)), [], name) + + def test_the_batch_commit_plan(self) -> None: + verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID) + self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts) + self.c.current.touched.add(mr.records_path("fake")) + plan = self.c.plan_commits(set()) + self.assertEqual(plan, [ + ("chore: add campaign measurements", ["measurements/fake.jsonl", "measurements/findings.jsonl"]), + ("chore: apply tested fields to fake", ["emulators/fake.yml"])]) + for subject, _ in plan: + self.assertEqual(campaign.autorun.check_subject(subject), [], subject) + # A file someone had changed before the pass is left to them. + self.assertEqual(self.c.plan_commits({"emulators/fake.yml"})[1:], []) + out = io.StringIO() + with contextlib.redirect_stdout(out): + self.assertEqual(self.c.commit_batch(dry_run=True), []) + self.assertIn("would commit 'chore: add campaign measurements'", out.getvalue()) + + +class Registry(unittest.TestCase): + def test_no_profile_takes_a_campaign_file_name(self) -> None: + stems = {p.stem for p in (REPO_ROOT / "emulators").glob("*.yml")} + self.assertEqual(stems & set(mr.CAMPAIGN_FILES), set()) + + def test_content_list(self) -> None: + content = campaign.load_content(campaign.CONTENT_FILE) + systems = {s for p in (REPO_ROOT / "emulators").glob("*.yml") + for s in (yaml.safe_load(p.read_text()) or {}).get("systems") or ()} + self.assertTrue(content) + for system, items in content.items(): + self.assertIn(system, systems) + for item in items: + self.assertIn(item.catalog, ("redump", "no-intro")) + self.assertGreater(len(item.reason), 40) + self.assertGreater(item.frames, item.frame_hash_every) + + def test_the_autorun_job(self) -> None: + jobs = campaign.autorun.load_jobs() + job = jobs["campaign"] + self.assertTrue(job.lock and job.worktree and job.schedule) + self.assertEqual(job.gate[0].argv, ("python", "scripts/campaign.py", "gate")) + self.assertEqual({p for rule in job.commits for p in rule.paths}, + {"measurements/*.jsonl", "emulators/*.yml"}) + self.assertGreater(job.schedule, jobs["nightly"].schedule) + + def test_autorun_tells_its_steps_the_lock_is_held(self) -> None: + job = campaign.autorun.load_jobs()["campaign"] + self.assertEqual(campaign.autorun.job_env(job)[campaign.LOCK_HELD_ENV], "1") + + +class SitePage(unittest.TestCase): + def test_last_pass_and_open_findings(self) -> None: + summary = { + "last_run": {"ended_at": "2026-10-27T10:00:00+00:00", "result": "complete", "detail": "", + "counts": {"done": 12, "failed": 1, "pending": 3}, + "pending_reasons": {"standalone": 3}, + "failed": [{"emulator": "fake", "check": "fake/x/y/bench", "reason": "timeout"}]}, + "passes": 1, "applied": 2, + "open": [{"emulator": "fake", "file": "bios.bin", "field": "required", "status": "open", + "profile_says": True, "run_shows": False, "statement": "boots without it", + "evidence": ["2026-10-27-aaaaaaaaaaaa"], "links": ["https://x/l#L1"]}], + } + page = "\n".join(sitemeasurements.campaign_section(summary)) + self.assertIn("12 tests done, 1 failed, 3 waiting", page) + self.assertIn("| standalone | 3 |", page) + self.assertIn("the profile says `true`, the run shows `false`", page) + self.assertIn("[2026-10-27-aaaaaaaaaaaa](https://x/l#L1)", page) + self.assertEqual(sitemeasurements.campaign_section({"last_run": None}), []) + + +@unittest.skipUnless(shutil.which("cc") and shutil.which("strace"), "no C compiler or no strace") +class FakeCorePasses(Workspace): + """Whole passes against a core compiled from FAKE_CORE.""" + + @classmethod + def setUpClass(cls) -> None: + TMP.mkdir(parents=True, exist_ok=True) + cls.build = Path(tempfile.mkdtemp(dir=TMP)) + source = cls.build / "fake.c" + source.write_text(FAKE_CORE, encoding="utf-8") + cls.core = cls.build / "fake_libretro.so" + subprocess.run(["cc", "-shared", "-fPIC", "-o", str(cls.core), str(source)], check=True) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.build, ignore_errors=True) + + def passing(self, env: dict[str, str] | None = None, core: Path | None = None, + **settings: object) -> campaign.Campaign: + self.put_game() + # The core_probe threshold follows the host: a test never waits on the real load. + host = campaign.Host(loadavg=lambda: (0.0, 0.0, 0.0), free_gb=lambda path: 500.0, + cpu_count=1_000_000) + extra = env or {} + + def run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess: + check = bool(kwargs.pop("check", False)) + return subprocess.run(argv, check=check, env={**os.environ, **extra}, **kwargs) + host.run = run + values = {"bench_warmup": 2, "bench_frames": 4, "check_repeat": 2, "probe_timeout": 30, + "delays": (1,), "echo": False, **settings} + c = campaign.Campaign(self.paths, campaign.Settings(**values), host, + lock_path=self.tmp / "auto.lock") + so = core or self.core + c.unit_cores = lambda *a, **k: [campaign.Core("fake", so, campaign.file_sha256(so))] + return c + + def test_a_stand_in_is_proven_and_written(self) -> None: + c = self.passing({"FAKE_HLE": "1"}) + current = c.run(set(), None) + self.assertEqual(current.result, "complete") + records = [s.record for s in mr.read_records(self.root)] + kinds = sorted(r["subject"].split("-")[0] for r in records) + self.assertEqual(kinds, ["alias", "bench", "boot", "files"]) + boot = next(r for r in records if r["subject"].startswith("boot")) + alias = next(r for r in records if r["subject"].startswith("alias")) + entry = yaml.safe_load((self.root / "emulators" / "fake.yml").read_text())["files"][0] + self.assertEqual(entry["hle_fallback"], True) + self.assertEqual(entry["evidence"], {"aliases": [alias["id"]], "hle_fallback": [boot["id"]]}) + statuses = {(f["field"], f["status"]) for f in mr.read_campaign(self.root, mr.FINDINGS)} + self.assertEqual(statuses, {("hle_fallback", "applied"), ("required", "open"), + ("aliases", "applied")}) + runs = mr.read_campaign(self.root, mr.CAMPAIGN_RUNS) + self.assertEqual(runs[-1]["counts"]["done"], 4) + # The game is pinned by nobody but was not fetched by the campaign: kept. + self.assertTrue(self.game.is_dir()) + + # A second pass on the same core and profile runs nothing. + again = self.passing({"FAKE_HLE": "1"}) + self.assertEqual(again.run(set(), None).records, []) + + # A new build of the core runs everything again. + rebuilt = self.build / "fake2_libretro.so" + shutil.copy2(self.core, rebuilt) + with rebuilt.open("ab") as handle: + handle.write(b"\0") + third = self.passing({"FAKE_HLE": "1"}, core=rebuilt) + # BIOS.BIN is declared now, so no alias run follows the boot test. + self.assertEqual(len(third.run(set(), None).records), 3) + + def test_no_stand_in_settles_required_and_finds_an_alias(self) -> None: + c = self.passing() + c.run(set(), None) + entry = yaml.safe_load((self.root / "emulators" / "fake.yml").read_text())["files"][0] + self.assertEqual((entry["required"], entry["hle_fallback"]), (True, False)) + self.assertIn("BIOS.BIN", entry["aliases"]) + self.assertEqual(sorted(entry["evidence"]), ["aliases", "hle_fallback", "required"]) + alias = [f for f in mr.read_campaign(self.root, mr.FINDINGS) if f["field"] == "aliases"] + self.assertEqual(alias[0]["status"], "applied") + + def test_a_hung_core_is_killed_and_the_pass_moves_on(self) -> None: + c = self.passing({"FAKE_HANG": "1"}, probe_timeout=2) + c.run(set(), None) + checks = c.state["checks"]["fake"] + self.assertTrue(checks) + self.assertEqual({entry["status"] for entry in checks.values()}, {"failed"}) + self.assertEqual({entry["reason"] for entry in checks.values()}, {"timeout"}) + self.assertEqual(mr.read_records(self.root), []) + + def test_missing_content_leaves_the_checks_pending(self) -> None: + c = self.passing(romfetch=str(self.tmp / "no-romfetch")) + shutil.rmtree(self.game) + c.run(set(), None) + reasons = {entry["reason"] for entry in c.state["checks"]["fake"].values()} + self.assertEqual(len(reasons), 1) + self.assertIn("romfetch not found", reasons.pop()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index 542511db1..49399ad06 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -154,6 +154,12 @@ class ParsingTests(unittest.TestCase): self.assertEqual(described, f"{library.name} sha256:{core_probe.file_digest(library, 'sha256')}") self.assertEqual(core_probe.describe_env_value("2000-01-01 00:00:00"), "2000-01-01 00:00:00") + def test_uniform_frame(self) -> None: + self.assertTrue(core_probe.uniform_frame(b"", 2)) + self.assertTrue(core_probe.uniform_frame(b"\x01\x02" * 8, 2)) + self.assertFalse(core_probe.uniform_frame(b"\x01\x02" * 7 + b"\x01\x03", 2)) + self.assertFalse(core_probe.uniform_frame(b"\x00\x00\x00\x00\x00\x00\x00\x01", 4)) + def test_cue_tracks(self) -> None: with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as tmp: cue = Path(tmp, "d.cue") @@ -265,8 +271,8 @@ class FakeCoreTests(unittest.TestCase): for p in record_dir.rglob("*") if p.is_file()], ["measurements/fake.jsonl"]) line = json.loads((record_dir / "measurements" / "fake.jsonl").read_text()) self.assertEqual(measurement_records.check_line(line), []) - # The line is the record the run printed, with its id. - self.assertEqual({k: v for k, v in line.items() if k != "id"}, record) + # The line is the record the run printed, id included. + self.assertEqual(line, record) try: from jsonschema import Draft202012Validator except ImportError: