#!/usr/bin/env python3 """Run the real-condition campaign unattended: cores, content, checks, rules. For each emulator profile in scope, the campaign takes the core (the libretro x86_64 Linux nightly, else a build of the profile's pinned commit), the test content of each of its systems (tests/campaign/content.yml, fetched through romfetch when it is not on disk), and runs core_probe.py: files every declared file the collection holds, staged; which ones the core opens boot the content with no file against the content with the declared file, compared on the frames the core shows path a declared file staged where the core looked for it alias a declared file staged under a name the core looked for variants each .variants/ copy of a file the core opened bench the content timed, warm-up excluded, 5 repetitions Every run is a record in measurements/.jsonl. The fixed rule table (campaign_rules.py) turns records into verdicts. A verdict the table settles is written into the profile with the record ids as `evidence:`. Any other goes to measurements/findings.jsonl, where the site lists it as open. Each pass adds one line to measurements/campaign-runs.jsonl. State is .cache/campaign/state.json of the main checkout: per check done, failed or pending with the reason, and the core sha256 and profile digest it ran against. A pass skips what is done unless the core build or the profile changed. The campaign holds the host-wide automation lock, waits for an idle host, stops cleanly on a busy host or a full disk, and resumes on the next pass. Under autorun (`campaign` job) the lock is the job's. Usage: python scripts/campaign.py run [--only CORE,...] [--system ID] [--budget 6h] python scripts/campaign.py run --dry-run python scripts/campaign.py run --commit [--commit-every N] python scripts/campaign.py status [--json] python scripts/campaign.py findings [--all] [--json] python scripts/campaign.py gate python scripts/campaign.py install-timer [--enable] """ from __future__ import annotations import argparse import contextlib import datetime as dt import hashlib import io import json import os import re import shutil import signal import stat import subprocess import sys import tarfile import time import urllib.error import urllib.request import zipfile from collections.abc import Callable, Iterator, Sequence from dataclasses import dataclass, field from pathlib import Path import yaml sys.path.insert(0, str(Path(__file__).resolve().parent)) import autorun import campaign_profiles import campaign_rules as rules import core_probe import measurement_records as mr ROOT = Path(__file__).resolve().parent.parent CONTENT_FILE = ROOT / "tests" / "campaign" / "content.yml" BUILDBOT = "https://buildbot.libretro.com/nightly/linux/x86_64/latest/" LOCK_HELD_ENV = autorun.LOCK_HELD_ENV ROMFETCH_ENV = "RETROBIOS_ROMFETCH" STATE_VERSION = 1 DONE = "done" FAILED = "failed" PENDING = "pending" CRASH_ATTEMPTS = 3 NETWORK_DELAYS = (10, 60, 300) MAX_ALIAS_CANDIDATES = 6 BENCH_WARMUP = 600 BENCH_FRAMES = 1800 BENCH_REPEAT = 5 CHECK_REPEAT = 2 PROBE_TIMEOUT = 900 BUILD_TIMEOUT = 3600 LOG_DAYS = 30 ROMFETCH_TRANSIENT = 3 EXIT_REFUSED = 3 class Pause(RuntimeError): """The host cannot run measurements now: the pass stops and resumes later.""" class Unavailable(RuntimeError): """A core or a piece of content the campaign cannot get.""" # -------------------------------------------------------------------------- # Small helpers # -------------------------------------------------------------------------- def now_iso() -> str: return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat() def slug(text: str) -> str: return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") or "x" def parse_budget(text: str) -> float: match = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([smh]?)\s*", text or "") if not match: raise argparse.ArgumentTypeError(f"--budget {text!r}: a number with s, m or h") return float(match[1]) * {"": 1, "s": 1, "m": 60, "h": 3600}[match[2]] def file_sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest() def file_sha1(path: Path) -> str: digest = hashlib.sha1() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1 << 20), b""): digest.update(chunk) return digest.hexdigest() def profile_digest(profile: dict) -> str: """What a check depends on in a profile: its entries, evidence left out, so the campaign's own evidence never makes it run again.""" entries = [{k: v for k, v in entry.items() if k != "evidence"} for entry in profile.get("files") or []] body = {"files": entries, "cores": profile.get("cores"), "type": profile.get("type"), "systems": profile.get("systems")} return hashlib.sha1(mr.canonical(body)).hexdigest() def entry_systems(entry: dict, profile: dict) -> list[str]: if entry.get("systems"): return list(entry["systems"]) if entry.get("system"): return [entry["system"]] return list(profile.get("systems") or []) def entry_dest(entry: dict) -> str: """Where the core reads the entry, relative to its system directory.""" path = entry.get("path") if isinstance(path, str) and path and not path.endswith("/"): return path return entry["name"] def region_matches(entry: dict, region: str | None) -> bool: regions = entry.get("region") if not regions or not region: return True if isinstance(regions, str): regions = [regions] return region in regions or "world" in regions def is_bios(entry: dict) -> bool: return entry.get("category", "bios") == "bios" and entry.get("type") != "directory" # -------------------------------------------------------------------------- # Host # -------------------------------------------------------------------------- def run_group(argv: Sequence[str], *, timeout: float | None = None, check: bool = False, capture_output: bool = False, **kwargs: object) -> subprocess.CompletedProcess: """subprocess.run in a process group of its own, the whole group killed on a timeout or an interruption: a hung core never outlives its run.""" if capture_output: kwargs.setdefault("stdout", subprocess.PIPE) kwargs.setdefault("stderr", subprocess.PIPE) with subprocess.Popen(list(argv), start_new_session=True, **kwargs) as proc: try: out, err = proc.communicate(timeout=timeout) except BaseException: with contextlib.suppress(ProcessLookupError): os.killpg(proc.pid, signal.SIGKILL) proc.communicate() raise result = subprocess.CompletedProcess(list(argv), proc.returncode, out, err) if check: result.check_returncode() return result @dataclass class Host: """What the campaign reads from the machine and the network, replaced in tests.""" loadavg: Callable[[], tuple[float, float, float]] = os.getloadavg sleep: Callable[[float], None] = time.sleep clock: Callable[[], float] = time.monotonic cpu_count: int = os.cpu_count() or 1 free_gb: Callable[[Path], float] = autorun.free_gb poll: float = 60.0 run: Callable[..., subprocess.CompletedProcess] = run_group urlopen: Callable[..., object] = urllib.request.urlopen @property def max_load(self) -> float: return self.cpu_count / 4 @dataclass class Settings: budget: float = 6 * 3600 max_load_wait: float = 3600 min_free_gb: float = 30 cache_cap_gb: float = 20 probe_timeout: int = PROBE_TIMEOUT check_repeat: int = CHECK_REPEAT bench_repeat: int = BENCH_REPEAT bench_warmup: int = BENCH_WARMUP bench_frames: int = BENCH_FRAMES romfetch: str = "" network: bool = True delays: tuple[float, ...] = NETWORK_DELAYS echo: bool = True # Third-party code (a nightly core, a project's Makefile) runs only under # bubblewrap unless this is set. allow_unsandboxed: bool = False @dataclass class Paths: """Where the campaign writes: records and profiles in the checkout it runs from, state, cores, logs and content in the main checkout's .cache.""" root: Path home: Path content: Path db: Path content_file: Path = CONTENT_FILE @property def state(self) -> Path: return self.home / "state.json" @property def logs(self) -> Path: return self.home / "logs" @property def emulators(self) -> Path: return self.root / "emulators" @classmethod def default(cls) -> Paths: main = autorun.main_checkout() return cls(root=ROOT, home=main / ".cache" / "campaign", content=main / ".cache" / "test-content", db=ROOT / "database.json") ROMFETCH_PASSED = ("XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME") def child_env(romfetch: bool = False) -> dict[str, str]: """What a program the campaign starts inherits: autorun's allowlist, never a token or a password. romfetch also gets its own settings (ROMFETCH_*), its account staying in its config file.""" names: list[str] = [] if romfetch: names = [k for k in os.environ if k.startswith("ROMFETCH_")] + list(ROMFETCH_PASSED) return autorun.safe_env(names) # Host-side git on a third-party project. It only ever touches bare mirrors # this campaign created: no working tree, never bound into a sandbox. No user # or system config is read, https is the only transport, and the overrides # blank every hook, filter, proxy and ssh command a config could name. GIT_SAFE = ( "-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", "-c", "core.sshCommand=", "-c", "core.gitProxy=", "-c", "core.askPass=", "-c", "credential.helper=", "-c", "protocol.allow=never", "-c", "protocol.https.allow=always", "-c", "filter.lfs.smudge=", "-c", "filter.lfs.clean=", "-c", "filter.lfs.process=", "-c", "filter.lfs.required=false", "-c", "core.attributesFile=/dev/null", ) MIRROR_URL = re.compile(r"^https://[A-Za-z0-9.-]+(?::\d+)?/[A-Za-z0-9._~/+-]+$") def host_git(*args: str) -> list[str]: return ["git", *GIT_SAFE, *args] def git_env() -> dict[str, str]: env = child_env() env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull, GIT_TERMINAL_PROMPT="0", GIT_ALLOW_PROTOCOL="https", GIT_ATTR_NOSYSTEM="1") return env def blank_filters(repo: Path) -> list[str]: """-c overrides emptying every filter driver the repository's config, includes followed, defines: git archive runs smudge filters.""" config = repo / "config" if not config.is_file(): return [] listing = run_group(["git", "config", "--file", str(config), "--includes", "--name-only", "--get-regexp", r"^filter\."], capture_output=True, env=git_env(), timeout=60).stdout.decode("utf-8", "replace") drivers = sorted({line.rsplit(".", 1)[0] for line in listing.splitlines() if line.count(".") >= 2}) out = [] for driver in drivers: for key in ("smudge", "clean", "process"): out += ["-c", f"{driver}.{key}="] out += ["-c", f"{driver}.required=false"] return out def git_out(*args: str, timeout: int = 1800) -> bytes: extra = blank_filters(Path(args[1])) if len(args) > 1 and args[0] == "-C" else [] return run_group(["git", *GIT_SAFE, *extra, *args], capture_output=True, check=True, env=git_env(), timeout=timeout).stdout def submodule_url(base: str, url: str) -> str: """A .gitmodules URL, relative ones read against the parent's URL.""" if url.startswith(("./", "../")): parts = base.rstrip("/").removesuffix(".git").split("/") for piece in url.split("/"): if piece == "..": parts.pop() elif piece not in (".", ""): parts.append(piece) url = "/".join(parts) return url def extract_tar(blob: bytes, target: Path) -> None: """A git archive unpacked into target: no absolute path, no .., no link that leaves target.""" target.mkdir(parents=True, exist_ok=True) with tarfile.open(fileobj=io.BytesIO(blob)) as handle: if hasattr(tarfile, "data_filter"): handle.extractall(target, filter="data") return for member in handle.getmembers(): name = Path(member.name) link = Path(member.linkname) if name.is_absolute() or ".." in name.parts or member.islnk() or ( member.issym() and (link.is_absolute() or ".." in link.parts)): raise Unavailable(f"core: unsafe archive member {member.name}") handle.extractall(target) def export_commit(mirror: Callable[[str], Path], url: str, commit: str, dest: Path) -> None: """The tree of commit, and of each submodule at the commit the tree records, as plain files in a fresh dest. Every object comes from a bare mirror (mirror(url) fetches into it), never from a working tree.""" if dest.exists(): shutil.rmtree(dest) def one(repo_url: str, sha: str, target: Path) -> None: if not re.fullmatch(r"[0-9a-f]{40}", sha): raise Unavailable(f"core: {sha!r} is not a commit id") bare = mirror(repo_url) try: git_out("-C", str(bare), "cat-file", "-e", f"{sha}^{{commit}}") except subprocess.CalledProcessError: git_out("-C", str(bare), "fetch", "--quiet", repo_url, sha) extract_tar(git_out("-C", str(bare), "archive", "--format=tar", sha), target) links = {} for entry in git_out("-C", str(bare), "ls-tree", "-r", "-z", sha).split(b"\0"): if entry.startswith(b"160000 commit "): meta, path = entry.split(b"\t", 1) links[path.decode()] = meta.split()[2].decode() if not links: return try: config = git_out("-C", str(bare), "config", "--blob", f"{sha}:.gitmodules", "-z", "--get-regexp", r"^submodule\..*\.(path|url)$").decode() except subprocess.CalledProcessError as exc: raise Unavailable(f"core: {repo_url} has submodules and no readable .gitmodules") from exc paths: dict[str, str] = {} urls: dict[str, str] = {} for item in filter(None, config.split("\0")): key, _, value = item.partition("\n") name, field_ = key[len("submodule."):].rsplit(".", 1) (paths if field_ == "path" else urls)[name] = value for name, path in paths.items(): if path in links and name in urls: one(submodule_url(repo_url, urls[name]), links[path], target / path) one(url, commit, dest) def built_core(tree: Path) -> Path: """The newest *_libretro.so a build left: a regular file inside the tree, never a link a build could point at a host file.""" root = tree.resolve() found = [] for path in tree.rglob("*_libretro.so"): info = path.lstat() if stat.S_ISREG(info.st_mode) and path.resolve().is_relative_to(root): found.append((info.st_mtime, path)) if not found: raise Unavailable(f"core: the build left no *_libretro.so regular file in {tree}") return max(found)[1] def copy_regular(src: Path, dest: Path) -> None: """Copy src only when it is a regular file, never through a link.""" fd = os.open(src, os.O_RDONLY | os.O_NOFOLLOW) with os.fdopen(fd, "rb") as handle: if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): raise Unavailable(f"core: {src} is not a regular file") dest.parent.mkdir(parents=True, exist_ok=True) with dest.open("wb") as out: shutil.copyfileobj(handle, out) @contextlib.contextmanager def working_directory(path: Path) -> Iterator[None]: """contextlib.chdir, which Python 3.9 lacks.""" before = Path.cwd() os.chdir(path) try: yield finally: os.chdir(before) @contextlib.contextmanager def host_lock(lock_path: Path) -> Iterator[None]: """The automation lock, unless the autorun job running us holds it.""" if os.environ.get(LOCK_HELD_ENV): yield return with autorun.global_lock(lock_path): yield # -------------------------------------------------------------------------- # State # -------------------------------------------------------------------------- def load_state(path: Path) -> dict: try: state = json.loads(path.read_text(encoding="utf-8")) except FileNotFoundError: state = {} state.setdefault("version", STATE_VERSION) for key in ("checks", "cores", "content", "builds", "emulators"): state.setdefault(key, {}) return state def save_state(path: Path, state: dict) -> None: autorun.save_state(path, state) def should_run(previous: dict | None, core_sha: str, digest: str) -> bool: """A check runs again when it never ran, is pending, or ran against another core build or another version of the profile.""" if not previous: return True if previous.get("status") == PENDING: return True return previous.get("core_sha256") != core_sha or previous.get("profile_digest") != digest # -------------------------------------------------------------------------- # Content: what each system is tested with, and where it comes from # -------------------------------------------------------------------------- @dataclass(frozen=True) class Content: system: str romfetch_system: str name: str catalog: str reason: str region: str | None load: str frames: int frame_hash_every: int pinned: bool boot_file: str | None @property def key(self) -> str: return f"{self.romfetch_system}/{self.name}" def load_content(path: Path) -> dict[str, list[Content]]: data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} out: dict[str, list[Content]] = {} for system, spec in (data.get("systems") or {}).items(): for item in spec.get("content") or (): out.setdefault(system, []).append(Content( system=system, romfetch_system=str(spec["romfetch"]), name=str(item["name"]), catalog=str(item["catalog"]), reason=str(item["reason"]).strip(), region=item.get("region"), load=str(item.get("load") or ""), frames=int(item.get("frames", 3600)), frame_hash_every=int(item.get("frame_hash_every", 10)), pinned=bool(item.get("pinned", False)), boot_file=spec.get("boot_file"), )) return out def load_file(folder: Path, load: str) -> Path: """The file handed to the core.""" if load == "cue": cues = sorted(folder.glob("*.cue")) if len(cues) != 1: raise Unavailable(f"{folder.name}: {len(cues)} .cue files, one expected") return cues[0] if load: return folder / load files = [p for p in sorted(folder.iterdir()) if p.is_file() and p.name != "identity.json"] if len(files) != 1: raise Unavailable(f"{folder.name}: {len(files)} files, name one with load:") return files[0] def verify_identity(folder: Path) -> dict: """The identity.json romfetch wrote, each listed file hashed again.""" try: identity = json.loads((folder / "identity.json").read_text(encoding="utf-8")) except FileNotFoundError as exc: raise Unavailable(f"{folder}: no identity.json") from exc if identity.get("status") != "verified": raise Unavailable(f"{folder.name}: identity status {identity.get('status')!r}") for item in identity.get("files") or (): path = folder / item["name"] if not path.is_file() or path.stat().st_size != item["size"] or file_sha1(path) != item["sha1"]: raise Unavailable(f"{folder.name}: {item['name']} does not match its identity") return identity def folder_size(folder: Path) -> int: return sum(p.stat().st_size for p in folder.rglob("*") if p.is_file()) class ContentStore: """Content on disk, its users counted, the least recently used evicted past the cap. Only what the campaign fetched is ever deleted, and never a pinned entry.""" def __init__(self, campaign: Campaign) -> None: self.c = campaign self.state: dict = campaign.state["content"] self.refs: dict[str, set[str]] = {} self.verified: set[str] = set() def folder(self, content: Content) -> Path: return self.c.paths.content / content.romfetch_system / content.name def hold(self, content: Content, user: str) -> None: self.refs.setdefault(content.key, set()).add(user) def get(self, content: Content) -> Path: """The folder of the content, fetched when absent. Unavailable when not.""" folder = self.folder(content) entry = self.state.setdefault(content.key, {"fetched_by_campaign": False}) entry["pinned"] = content.pinned if not (folder / "identity.json").is_file(): self.evict(keep={content.key}) self.fetch(content) entry["fetched_by_campaign"] = True if content.key not in self.verified: identity = verify_identity(folder) entry["identity"] = {"catalog": identity.get("catalog"), "dat": identity.get("dat"), "entry": identity.get("entry")} self.verified.add(content.key) entry["size"] = folder_size(folder) entry["last_used"] = now_iso() return folder def romfetch(self) -> str | None: configured = self.c.settings.romfetch or os.environ.get(ROMFETCH_ENV, "") if configured: return configured if Path(configured).is_file() or shutil.which(configured) else None return shutil.which("romfetch") def fetch(self, content: Content) -> None: command = self.romfetch() if not command: raise Unavailable(f"content: romfetch not found, {content.catalog} entry {content.name!r}") if not self.c.settings.network: raise Unavailable(f"content: offline, {content.catalog} entry {content.name!r}") argv = [command, "get", content.romfetch_system, content.name, "--json", "-q", "--content-dir", str(self.c.paths.content)] for attempt, delay in enumerate((*self.c.settings.delays, None)): proc = self.c.host.run(argv, capture_output=True, text=True, check=False, timeout=6 * 3600, env=child_env(romfetch=True)) if proc.returncode == 0: self.c.log(f"romfetch {content.name}: {proc.stdout.strip()[:300]}") return code = "unknown" with contextlib.suppress(ValueError, AttributeError): code = json.loads(proc.stdout)["error"]["code"] if proc.returncode != ROMFETCH_TRANSIENT or delay is None: raise Unavailable(f"content: romfetch exit {proc.returncode} ({code}) " f"for {content.name!r}") self.c.log(f"romfetch {content.name}: exit {proc.returncode} ({code}), " f"retry {attempt + 1} in {delay}s") self.c.host.sleep(delay) def release(self, content: Content, user: str) -> None: users = self.refs.get(content.key, set()) users.discard(user) if not users: self.drop(content.key) def deletable(self, key: str) -> bool: entry = self.state.get(key) or {} return bool(entry.get("fetched_by_campaign")) and not entry.get("pinned") def drop(self, key: str) -> bool: if not self.deletable(key) or self.refs.get(key): return False folder = self.c.paths.content / key if folder.is_dir(): shutil.rmtree(folder) self.c.log(f"content {key}: deleted, no check left needs it") self.state.pop(key, None) self.verified.discard(key) return True def evict(self, keep: set[str]) -> list[str]: """Least recently used first, until the campaign's content fits the cap.""" cap = self.c.settings.cache_cap_gb * 1024 ** 3 held = sorted((entry.get("last_used", ""), key) for key, entry in self.state.items() if self.deletable(key) and key not in keep) total = sum(entry.get("size", 0) for key, entry in self.state.items() if self.deletable(key)) evicted = [] for _, key in held: if total <= cap: break size = self.state[key].get("size", 0) folder = self.c.paths.content / key if folder.is_dir(): shutil.rmtree(folder) self.state.pop(key, None) self.verified.discard(key) total -= size evicted.append(key) self.c.log(f"content {key}: evicted, the cache is over {self.c.settings.cache_cap_gb} GB") return evicted # -------------------------------------------------------------------------- # Cores # -------------------------------------------------------------------------- @dataclass(frozen=True) class Core: name: str path: Path sha256: str artifact: str | None = None repo: str | None = None commit: str | None = None build_command: str | None = None def safe_extract_so(archive: Path, member: str, target: Path) -> None: with zipfile.ZipFile(archive) as handle: names = handle.namelist() if member not in names: raise Unavailable(f"{archive.name}: no {member} inside ({', '.join(names[:5])})") target.parent.mkdir(parents=True, exist_ok=True) scratch = target.with_suffix(".part") with handle.open(member) as src, scratch.open("wb") as dst: shutil.copyfileobj(src, dst) os.replace(scratch, target) class Cores: """Buildbot nightlies, downloaded once per build, else local builds of the profile's pinned commit.""" def __init__(self, campaign: Campaign) -> None: self.c = campaign self.listing: set[str] | None = None def retrying(self, what: str, call: Callable[[], object]) -> object: for attempt, delay in enumerate((*self.c.settings.delays, None)): try: return call() except (urllib.error.URLError, TimeoutError, ConnectionError, OSError) as exc: if isinstance(exc, urllib.error.HTTPError) and exc.code < 500 and exc.code != 429: raise Unavailable(f"{what}: HTTP {exc.code}") from exc if delay is None: raise Unavailable(f"{what}: {exc}") from exc self.c.log(f"{what}: {exc}, retry {attempt + 1} in {delay}s") self.c.host.sleep(delay) raise AssertionError("unreachable") def buildbot(self) -> set[str]: if self.listing is None: if not self.c.settings.network: self.listing = set() else: def read() -> bytes: with self.c.host.urlopen(BUILDBOT, timeout=120) as response: return response.read() page = self.retrying("buildbot listing", read).decode("utf-8", "replace") self.listing = set(re.findall(r"([A-Za-z0-9_]+)_libretro\.so\.zip", page)) return self.listing def names(self, profile: dict) -> list[str]: listing = self.buildbot() return sorted({str(c) for c in profile.get("cores") or () if str(c) in listing}) def nightly(self, name: str, profile: dict) -> Core: url = f"{BUILDBOT}{name}_libretro.so.zip" cached = self.c.state["cores"].get(name) or {} def head() -> dict: request = urllib.request.Request(url, method="HEAD") with self.c.host.urlopen(request, timeout=120) as response: return {"etag": response.headers.get("ETag"), "last_modified": response.headers.get("Last-Modified")} stamp = self.retrying(f"{name} nightly", head) target = self.c.paths.home / "cores" / name / f"{name}_libretro.so" if (cached.get("etag"), cached.get("last_modified")) != (stamp["etag"], stamp["last_modified"]) \ or not target.is_file() or file_sha256(target) != cached.get("sha256"): archive = target.parent / f"{name}_libretro.so.zip" archive.parent.mkdir(parents=True, exist_ok=True) def download() -> None: with self.c.host.urlopen(url, timeout=600) as response, archive.open("wb") as out: shutil.copyfileobj(response, out) self.retrying(f"{name} nightly", download) safe_extract_so(archive, f"{name}_libretro.so", target) archive.unlink() cached = {**stamp, "url": url, "sha256": file_sha256(target), "fetched_at": now_iso()} self.c.state["cores"][name] = cached self.c.log(f"core {name}: new nightly {cached['sha256'][:12]} ({stamp['last_modified']})") repo = profile.get("source") if isinstance(profile.get("source"), str) else None return Core(name, target, cached["sha256"], artifact=url, repo=repo if repo and repo.startswith("http") else None) def build(self, emulator: str, profile: dict) -> Core: """The profile's pinned commit, built once and kept.""" source, commit = profile.get("source"), profile.get("source_commit") if not (isinstance(source, str) and source.startswith("http") and isinstance(commit, str)): raise Unavailable("core: not on the buildbot, and no source and source_commit to build") record = self.c.state["builds"].setdefault(emulator, {}) if record.get("commit") == commit and record.get("error"): raise Unavailable(f"core: build of {commit[:12]} failed before: {record['error']}") out = self.c.paths.home / "builds" / emulator / commit built = sorted(out.glob("*_libretro.so")) if not built: if not self.c.settings.network: raise Unavailable("core: not on the buildbot, offline, nothing built yet") try: so, command = self.compile(source, commit, out) except Unavailable as exc: self.c.state["builds"][emulator] = {"commit": commit, "error": str(exc), "at": now_iso()} raise built = [so] record["command"] = command command = record.get("command") or "make" self.c.state["builds"][emulator] = {"commit": commit, "command": command, "so": built[0].name, "at": now_iso()} return Core(built[0].name.removesuffix("_libretro.so"), built[0], file_sha256(built[0]), repo=source, commit=commit, build_command=command) def mirror(self, url: str) -> Path: """The bare mirror of a repository, created on first use. Only host git with the overrides of GIT_SAFE ever touches it.""" if not MIRROR_URL.match(url): raise Unavailable(f"core: {url!r} is not an https repository URL") bare = self.c.paths.home / "mirrors" / f"{slug(url)}.git" if not (bare / "HEAD").is_file(): bare.parent.mkdir(parents=True, exist_ok=True) git_out("clone", "--bare", "--quiet", url, str(bare)) return bare def compile(self, source: str, commit: str, out: Path) -> tuple[Path, str]: """Fetch the commit into a bare mirror (host git, networked), export it and its submodules with git archive into a fresh folder, and run the project's make there under bubblewrap: no network, that folder the one writable path. Returns the .so and the command, with how it was isolated.""" log = self.c.paths.logs / "builds" / f"{slug(source)}-{commit[:12]}.log" log.parent.mkdir(parents=True, exist_ok=True) isolation = core_probe.sandbox_argv([])[1] if isolation != core_probe.SANDBOX and not self.c.settings.allow_unsandboxed: raise Unavailable(f"refused: no sandbox to build {source} in ({isolation}), " f"--allow-unsandboxed overrides") # Working-tree clones of earlier campaign versions are never reused. shutil.rmtree(self.c.paths.home / "src", ignore_errors=True) tree = self.c.paths.home / "build-trees" / f"{slug(source)}-{commit[:12]}" try: export_commit(self.mirror, source, commit, tree) except (OSError, subprocess.CalledProcessError, tarfile.TarError) as exc: raise Unavailable(f"core: exporting {commit[:12]} of {source} failed: {exc}") from exc makefile = "Makefile.libretro" if (tree / "Makefile.libretro").is_file() else "Makefile" jobs = str(max(1, (os.cpu_count() or 2) // 2)) command = ["make", "-f", makefile, f"-j{jobs}", "platform=unix"] prefix, isolation = core_probe.sandbox_argv([tree]) with log.open("a", encoding="utf-8") as handle: handle.write(f"$ {' '.join(command)} in {tree} ({isolation})\n") handle.flush() proc = self.c.host.run(prefix + command, cwd=tree, stdout=handle, stderr=subprocess.STDOUT, check=False, timeout=BUILD_TIMEOUT, env=child_env()) if proc.returncode != 0: tail = log.read_text(encoding="utf-8", errors="replace").strip().splitlines()[-3:] raise Unavailable(f"core: make exited {proc.returncode}: {' | '.join(tail)} (log {log})") built = built_core(tree) copy_regular(built, out / built.name) shutil.rmtree(tree, ignore_errors=True) return out / built.name, f"{' '.join(command)} on git archive (isolation: {isolation})" # -------------------------------------------------------------------------- # Probe runs # -------------------------------------------------------------------------- @dataclass class ProbeResult: status: str record: dict | None = None reason: str = "" log: str = "" @dataclass class Unit: """One core of one emulator profile, with its systems' content.""" emulator: str profile: dict core_name: str | None = None pending: str = "" # A game engine or utility profile: run when the buildbot has its core, # otherwise it is a standalone program. buildbot_only: bool = False @dataclass class Pass: started: str records: list[str] = field(default_factory=list) findings: dict[str, int] = field(default_factory=dict) touched: set[str] = field(default_factory=set) result: str = "complete" detail: str = "" class Campaign: def __init__(self, paths: Paths, settings: Settings | None = None, host: Host | None = None, probe_script: Path | None = None, lock_path: Path | None = None) -> None: self.paths = paths self.settings = settings or Settings() self.host = host or Host() self.probe_script = probe_script or ROOT / "scripts" / "core_probe.py" self.lock_path = lock_path or Path(os.environ.get(autorun.LOCK_ENV) or autorun.DEFAULT_LOCK) self.state = load_state(paths.state) self.cores = Cores(self) self.store = ContentStore(self) self.content = load_content(paths.content_file) self.deadline = 0.0 self.current: Pass | None = None self.dirty_at_start: set[str] = set() self._db: dict | None = None # -- logging ------------------------------------------------------------ def log(self, text: str) -> None: line = f"{now_iso()} {text}" path = self.paths.logs / f"{dt.date.today().isoformat()}.log" path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as handle: handle.write(line + "\n") if self.settings.echo: print(line, file=sys.stderr) # -- profiles and files ------------------------------------------------- def profiles(self) -> dict[str, dict]: out = {} for path in sorted(self.paths.emulators.glob("*.yml")): if path.name.startswith("_") or path.stem == "00_example": continue data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} if "emulator" in data: out[path.stem] = data return out @property def db(self) -> dict: if self._db is None: self._db = json.loads(self.paths.db.read_text(encoding="utf-8")) return self._db def local_file(self, entry: dict) -> Path | None: """The collection's copy of an entry, by its declared hashes or its name.""" from common import resolve_local_file hint = entry.get("path") if isinstance(entry.get("path"), str) else "" # The database names files relative to the checkout it describes. with working_directory(self.paths.root): path, status = resolve_local_file(entry, self.db, dest_hint=hint or "") if not path or status in ("not_found", "hash_mismatch"): return None local = Path(path) local = local if local.is_absolute() else self.paths.root / local return local if local.is_file() else None def variant_files(self, entry: dict, primary: Path) -> list[Path]: """The .variants/ copies of an entry's file the collection holds.""" names = [entry["name"], *(entry.get("aliases") or ())] out = [] index = self.db.get("indexes", {}).get("by_name", {}) for name in names: for sha1 in index.get(name, ()): path = (self.db["files"].get(sha1) or {}).get("path", "") if "/.variants/" in path: local = self.paths.root / path if local.is_file() and local.resolve() != primary.resolve() and local not in out: out.append(local) return out # -- guards --------------------------------------------------------------- def guard(self) -> None: """Raise Pause when the pass must stop: budget spent, disk full, host busy.""" if self.host.clock() >= self.deadline: raise Pause("budget") free = self.host.free_gb(self.paths.home if self.paths.home.exists() else self.paths.root) if free < self.settings.min_free_gb: raise Pause(f"disk: {free:.0f} GB free, the campaign stops under " f"{self.settings.min_free_gb:.0f} GB") waited = 0.0 while self.host.loadavg()[0] > self.host.max_load: if waited >= self.settings.max_load_wait: raise Pause(f"busy host: load stayed above {self.host.max_load:.1f} for " f"{int(waited)} s") self.host.sleep(self.host.poll) waited += self.host.poll # -- one core_probe invocation ------------------------------------------- def probe(self, argv: list[str], log_name: str) -> ProbeResult: """core_probe with retries for a crashing core. Never raises for the core's sake.""" command = [sys.executable, str(self.probe_script), *argv] log = self.paths.logs / dt.date.today().isoformat() / f"{log_name}.log" log.parent.mkdir(parents=True, exist_ok=True) repeats = argv.count("--vary") or 1 reps = int(argv[argv.index("--repeat") + 1]) if "--repeat" in argv else 1 budget = self.settings.probe_timeout * repeats * reps + 120 attempt = busy = 0 while attempt < CRASH_ATTEMPTS: self.guard() try: proc = self.host.run(command, capture_output=True, text=True, check=False, timeout=budget, cwd=self.paths.root, env=child_env()) except subprocess.TimeoutExpired: with log.open("a", encoding="utf-8") as handle: handle.write(f"$ {' '.join(command)}\nkilled after {budget} s\n") return ProbeResult(FAILED, reason="timeout", log=str(log)) with log.open("a", encoding="utf-8") as handle: handle.write(f"$ {' '.join(command)}\n# exit {proc.returncode}\n" f"{proc.stderr[-20000:]}\n") if proc.returncode == 0: try: return ProbeResult(DONE, record=json.loads(proc.stdout), log=str(log)) except ValueError: return ProbeResult(FAILED, reason="core_probe printed no record", log=str(log)) if proc.returncode == 3: # Refused on a busy host: wait for it to settle and run it again. busy += 1 if busy >= CRASH_ATTEMPTS: raise Pause("busy host: core_probe refused to record three times") continue attempt += 1 if proc.returncode == 2: return ProbeResult(FAILED, reason=f"core_probe refused the run: " f"{proc.stderr.strip()[-300:]}", log=str(log)) if "TimeoutExpired" in proc.stderr or "timed out after" in proc.stderr: return ProbeResult(FAILED, reason="timeout", log=str(log)) self.log(f"{log_name}: attempt {attempt} exited {proc.returncode}") return ProbeResult(FAILED, reason=f"crashed {CRASH_ATTEMPTS} times", log=str(log)) def probe_args(self, mode: str, core: Core, emulator: str, subject: str, content: Content | None, load: Path | None) -> list[str]: argv = [mode, "--core", str(core.path), "--record", str(self.paths.root), "--emulator", emulator, "--subject", subject, "--max-load", str(self.host.max_load), "--timeout", str(self.settings.probe_timeout), "--work-dir", str(self.paths.root / "tmp" / "core_probe")] if not self.settings.allow_unsandboxed: argv.append("--require-sandbox") if core.artifact: argv += ["--core-artifact", core.artifact] if core.repo: argv += ["--core-repo", core.repo] if core.commit: argv += ["--core-commit", core.commit] if core.build_command: argv += ["--build-command", core.build_command] if content and load: argv += ["--content", str(load), "--content-label", content.name] return argv # -- the checks ------------------------------------------------------------- def check_key(self, core: Core, content: Content, check: str) -> str: return f"{core.name}/{content.system}/{slug(content.name)}/{check}" def remember(self, emulator: str, key: str, result: ProbeResult, core: Core | None, digest: str) -> None: entry = {"status": result.status, "at": now_iso(), "profile_digest": digest, "core_sha256": core.sha256 if core else None} if result.reason: entry["reason"] = result.reason if result.log: entry["log"] = result.log if result.record: entry["record"] = result.record["id"] if self.current: self.current.records.append(result.record["id"]) self.current.touched.add(mr.records_path(emulator)) self.state["checks"].setdefault(emulator, {})[key] = entry def pending(self, emulator: str, key: str, reason: str, digest: str = "") -> None: self.state["checks"].setdefault(emulator, {})[key] = { "status": PENDING, "reason": reason, "at": now_iso(), "profile_digest": digest} def system_entries(self, profile: dict, system: str) -> list[tuple[int, dict]]: from validation import filter_files_by_mode libretro = filter_files_by_mode(profile.get("files") or [], standalone=False) keep = {id(e) for e in libretro} return [(i, e) for i, e in enumerate(profile.get("files") or []) if id(e) in keep and system in entry_systems(e, profile) and is_bios(e)] def boot_entry(self, entries: list[tuple[int, dict]], content: Content) -> tuple[int, dict] | None: held = [(i, e) for i, e in entries if region_matches(e, content.region) and self.local_file(e)] if content.boot_file: named = [(i, e) for i, e in held if content.boot_file in (e["name"], *(e.get("aliases") or ()))] if named: return named[0] held.sort(key=lambda item: (item[1].get("priority", 1 << 30), item[0])) return held[0] if held else None def run_unit_content(self, unit: Unit, core: Core, content: Content, digest: str) -> None: emulator, profile = unit.emulator, unit.profile checks = self.state["checks"].setdefault(emulator, {}) entries = self.system_entries(profile, content.system) boot = self.boot_entry(entries, content) keys = {name: self.check_key(core, content, name) for name in ("files", "bench")} if boot: keys["boot"] = self.check_key(core, content, f"boot:{boot[1]['name']}") todo = [k for k in keys.values() if should_run(checks.get(k), core.sha256, digest)] if not todo: return try: folder = self.store.get(content) load = load_file(folder, content.load) except Unavailable as exc: for key in todo: self.pending(emulator, key, str(exc), digest) return frames = ["--frames", str(content.frames), "--frame-hash-every", str(content.frame_hash_every)] stem = f"{core.name}-{slug(content.name)}" # files: every declared file the collection holds, at its place. staged: dict[str, tuple[int, dict, Path]] = {} for index, entry in entries: local = self.local_file(entry) if local and entry_dest(entry) not in staged: staged[entry_dest(entry)] = (index, entry, local) files_record = None if keys["files"] in todo: argv = self.probe_args("run", core, emulator, slug(f"files-{stem}"), content, load) + frames for dest, (_, _, local) in sorted(staged.items()): argv += ["--system-file", f"{dest}={local}"] result = self.probe(argv, f"{emulator}-{stem}-files") self.remember(emulator, keys["files"], result, core, digest) files_record = result.record elif checks.get(keys["files"], {}).get("status") == DONE: files_record = self.read_record(emulator, checks[keys["files"]].get("record")) boot_record = None if boot and keys["boot"] in todo: index, entry = boot local = self.local_file(entry) dest = entry_dest(entry) argv = self.probe_args("check", core, emulator, slug(f"boot-{entry['name']}-{stem}"), content, load) + frames argv += ["--vary", f"{dest}=", "--vary", f"{dest}={local}", "--observable", "frame_hashes", "--repeat", str(self.settings.check_repeat), "--claim", f"{core.name} reaches the game on {content.name} without {entry['name']}", "--method", (f"Two variants, {entry['name']} absent and {entry['name']} as the " f"collection holds it, each run {self.settings.check_repeat} times " f"from a fresh process for {content.frames} frames with no button " f"pressed. Every {content.frame_hash_every}th frame is hashed. The " f"runs without the file are compared with the runs with it on the " f"frames that are not one colour.")] result = self.probe(argv, f"{emulator}-{stem}-boot") self.remember(emulator, keys["boot"], result, core, digest) boot_record = result.record if boot_record: absent, present = (v["label"] for v in boot_record["variants"]) verdicts = rules.rule_bios_need(rules.boot_outcome(boot_record, absent, present), entry, boot_record["id"]) self.settle(emulator, content, core, index, entry, verdicts) self.follow_aliases(unit, core, content, load, frames, digest, index, entry, local, boot_record, absent) if files_record and keys["files"] in todo: self.read_files(unit, core, content, load, frames, digest, staged, files_record) if keys["bench"] in todo: argv = self.probe_args("bench", core, emulator, slug(f"bench-{stem}"), content, load) argv += ["--warmup", str(self.settings.bench_warmup), "--frames", str(self.settings.bench_frames), "--repeat", str(self.settings.bench_repeat), "--no-trace"] if boot: local = self.local_file(boot[1]) argv += ["--system-file", f"{entry_dest(boot[1])}={local}"] result = self.probe(argv, f"{emulator}-{stem}-bench") self.remember(emulator, keys["bench"], result, core, digest) def read_record(self, emulator: str, rid: str | None) -> dict | None: if not rid: return None path = self.paths.root / mr.records_path(emulator) if not path.is_file(): return None for line in path.read_text(encoding="utf-8").splitlines(): if f'"id":"{rid}"' in line: return json.loads(line) return None def read_files(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str], digest: str, staged: dict[str, tuple[int, dict, Path]], record: dict) -> None: """Which staged files the core opened: path follow-ups, variants, flags.""" emulator = unit.emulator rep = record["variants"][0]["repetitions"][0] if rep.get("files_opened") is None: return opened = {item["path"] for item in rep.get("files_opened") or ()} probed = list(rep.get("files_probed_absent") or ()) for dest, (index, entry, local) in sorted(staged.items()): if dest in opened: variants = self.variant_files(entry, local) if variants: self.run_variants(unit, core, content, load, frames, digest, index, entry, dest, local, variants) continue if not region_matches(entry, content.region): continue # A fallback is read only when the files before it are absent: # staged alone, it shows whether the core reads it at all. alone = self.follow_up(unit, core, content, load, frames, digest, entry, local, dest, f"alone:{entry['name']}") if alone is None: continue if dest in self.opened(alone, 1): continue probed_alone = [p for r in alone["variants"][0]["repetitions"] for p in r.get("files_probed_absent") or ()] others = [p for p in dict.fromkeys(probed + probed_alone) if p != dest and p.rsplit("/", 1)[-1].casefold() == dest.rsplit("/", 1)[-1].casefold()] moved = False for candidate in others[:MAX_ALIAS_CANDIDATES]: record_c = self.follow_up(unit, core, content, load, frames, digest, entry, local, candidate, f"path:{entry['name']}:{candidate}") if record_c and candidate in self.opened(record_c, 1): verdict = rules.rule_path(entry, dest, rules.Opened(candidate, record_c["id"])) if verdict: self.settle(emulator, content, core, index, entry, [verdict]) moved = True break if not moved: verdict = rules.rule_never_opened(entry, [record["id"], alone["id"]], content.name) self.settle(emulator, content, core, index, entry, [verdict]) @staticmethod def opened(record: dict, position: int) -> set[str]: out: set[str] = set() for rep in record["variants"][position]["repetitions"]: out |= {item["path"] for item in rep.get("files_opened") or ()} return out def follow_up(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str], digest: str, entry: dict, local: Path, dest: str, check: str) -> dict | None: """The file staged at dest alone, against nothing there.""" key = self.check_key(core, content, check) previous = self.state["checks"].get(unit.emulator, {}).get(key) if not should_run(previous, core.sha256, digest): return self.read_record(unit.emulator, (previous or {}).get("record")) stem = f"{core.name}-{slug(content.name)}" argv = self.probe_args("check", core, unit.emulator, slug(f"{check}-{stem}"), content, load) argv += frames + ["--vary", f"{dest}=", "--vary", f"{dest}={local}", "--observable", "frame_hashes", "--repeat", "1", "--claim", f"{core.name} opens {entry['name']} as {dest}", "--method", (f"{entry['name']} staged as {dest}, against nothing there, " f"{content.frames} frames of {content.name}. The files the " f"core opened are read from strace.")] result = self.probe(argv, f"{unit.emulator}-{stem}-{slug(check)}") self.remember(unit.emulator, key, result, core, digest) return result.record def follow_aliases(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str], digest: str, index: int, entry: dict, local: Path, record: dict, absent_label: str) -> None: """Names the core looked for, with the file absent, that no entry declares.""" declared = {e["name"] for e in unit.profile.get("files") or ()} declared |= {a for e in unit.profile.get("files") or () for a in e.get("aliases") or ()} suffix = Path(entry["name"]).suffix.casefold() dest = entry_dest(entry) folder = dest.rsplit("/", 1)[0] if "/" in dest else "" probed: list[str] = [] for rep in rules.variant(record, absent_label)["repetitions"]: for path in rep.get("files_probed_absent") or (): name = path.rsplit("/", 1)[-1] where = path.rsplit("/", 1)[0] if "/" in path else "" if (where == folder and name not in declared and Path(name).suffix.casefold() == suffix and path not in probed): probed.append(path) for candidate in probed[:MAX_ALIAS_CANDIDATES]: found = self.follow_up(unit, core, content, load, frames, digest, entry, local, candidate, f"alias:{entry['name']}:{candidate.rsplit('/', 1)[-1]}") if found and candidate in self.opened(found, 1): verdict = rules.rule_alias(self.reload_entry(unit, index), rules.Opened(candidate, found["id"])) if verdict: self.settle(unit.emulator, content, core, index, self.reload_entry(unit, index), [verdict]) def run_variants(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str], digest: str, index: int, entry: dict, dest: str, primary: Path, variants: list[Path]) -> None: key = self.check_key(core, content, f"variants:{entry['name']}") if not should_run(self.state["checks"].get(unit.emulator, {}).get(key), core.sha256, digest): return stem = f"{core.name}-{slug(content.name)}" argv = self.probe_args("check", core, unit.emulator, slug(f"variants-{entry['name']}-{stem}"), content, load) + frames for path in [primary, *variants]: argv += ["--vary", f"{dest}={path}"] argv += ["--observable", "frame_hashes", "--repeat", str(self.settings.check_repeat), "--claim", f"the copies of {entry['name']} change what {core.name} shows on {content.name}", "--method", (f"Each copy the collection holds staged as {dest}, each run " f"{self.settings.check_repeat} times for {content.frames} frames, " f"compared on the sampled frame hashes.")] result = self.probe(argv, f"{unit.emulator}-{stem}-variants") self.remember(unit.emulator, key, result, core, digest) if result.record: status = result.record["check"]["result"]["status"] verdict = rules.rule_variants(status, [v["label"] for v in result.record["variants"]], entry, result.record["id"]) if verdict: self.settle(unit.emulator, content, core, index, entry, [verdict]) # -- verdicts ----------------------------------------------------------------- def reload_entry(self, unit: Unit, index: int) -> dict: path = self.paths.emulators / f"{unit.emulator}.yml" unit.profile = yaml.safe_load(path.read_text(encoding="utf-8")) return unit.profile["files"][index] def settle(self, emulator: str, content: Content, core: Core, index: int, entry: dict, verdicts: Sequence[rules.Verdict]) -> None: """Apply what the table settles, queue the rest, resolve what now agrees.""" path = self.paths.emulators / f"{emulator}.yml" for verdict in verdicts: fresh = (yaml.safe_load(path.read_text(encoding="utf-8")).get("files") or [])[index] if fresh.get("name") != entry.get("name"): self.log(f"{emulator}: files[{index}] is no longer {entry.get('name')}, nothing applied") return says = rules.current(fresh, verdict.field) if verdict.field in rules.SETTLED_FIELDS \ else None agrees = verdict.field in rules.SETTLED_FIELDS and rules.agrees(fresh, verdict) if verdict.apply: updates: dict[str, object] = {} if not agrees or verdict.field not in fresh: updates[verdict.field] = verdict.value evidence = rules.merge_evidence(fresh.get("evidence"), verdict) if evidence != fresh.get("evidence"): updates["evidence"] = evidence if updates: try: changed = campaign_profiles.write_fields(path, index, updates) except campaign_profiles.ProfileEditError as exc: self.finding(emulator, content, core, fresh, verdict, "open", says, extra=f". Not written: {exc}") continue if changed and self.current: self.current.touched.add(path.relative_to(self.paths.root).as_posix()) if agrees: self.resolve(emulator, content, core, fresh, verdict, says) else: self.finding(emulator, content, core, fresh, verdict, "applied", says) continue if verdict.rule == "never_opened": self.finding(emulator, content, core, fresh, verdict, "flagged", None) elif agrees: self.resolve(emulator, content, core, fresh, verdict, says) else: self.finding(emulator, content, core, fresh, verdict, "open", says) def finding_key(self, emulator: str, content: Content, entry: dict, verdict: rules.Verdict) -> str: return f"{emulator}/{content.system}/{entry['name']}/{verdict.field}" def latest_findings(self) -> dict[str, dict]: latest: dict[str, dict] = {} for line in mr.read_campaign(self.paths.root, mr.FINDINGS): latest[line["key"]] = line return latest def finding(self, emulator: str, content: Content, core: Core, entry: dict, verdict: rules.Verdict, status: str, says: object, extra: str = "") -> None: key = self.finding_key(emulator, content, entry, verdict) previous = self.latest_findings().get(key) if previous and previous["status"] == status and previous["rule"] == verdict.rule \ and previous["run_shows"] == verdict.value and previous["profile_says"] == says: return line = mr.seal_line({ "schema_version": 1, "found_at": now_iso(), "key": key, "emulator": emulator, "system": content.system, "file": entry["name"], "field": verdict.field, "status": status, "rule": verdict.rule, "profile_says": says, "run_shows": verdict.value, "statement": verdict.statement + extra, "evidence": list(verdict.evidence), "core": {"file": core.path.name, "sha256": core.sha256, "library_version": self.library_version(emulator, verdict.evidence)}, "content": content.name, }, "found_at") path = self.paths.root / mr.records_path(mr.FINDINGS) path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as handle: handle.write(mr.line_text(line) + "\n") if self.current: self.current.findings[status] = self.current.findings.get(status, 0) + 1 self.current.touched.add(mr.records_path(mr.FINDINGS)) self.log(f"finding {status}: {key}: {verdict.statement}") def resolve(self, emulator: str, content: Content, core: Core, entry: dict, verdict: rules.Verdict, says: object) -> None: previous = self.latest_findings().get(self.finding_key(emulator, content, entry, verdict)) if previous and previous["status"] in ("open", "flagged"): self.finding(emulator, content, core, entry, verdict, "resolved", says) def library_version(self, emulator: str, ids: Sequence[str]) -> str | None: for rid in ids: record = self.read_record(emulator, rid) if record: return record["core"].get("library_version") return None # -- the pass ----------------------------------------------------------------- def units(self, only: set[str], system: str | None) -> list[Unit]: out = [] for emulator, profile in sorted(self.profiles().items()): cores = {str(c) for c in profile.get("cores") or ()} if only and emulator not in only and not (cores & only): continue systems = set(profile.get("systems") or ()) if system and system not in systems: continue kind = str(profile.get("type") or "") if kind in ("alias", "launcher", "test"): continue if "libretro" in kind or kind in ("game", "utility"): out.append(Unit(emulator, profile, buildbot_only="libretro" not in kind)) continue out.append(Unit(emulator, profile, pending="standalone")) return out def unit_contents(self, unit: Unit, system: str | None) -> list[Content]: systems = [s for s in unit.profile.get("systems") or () if not system or s == system] return [c for s in systems for c in self.content.get(s, ())] def run(self, only: set[str], system: str | None, dry_run: bool = False, commit_every: int = 0, commit: bool = False) -> Pass: self.deadline = self.host.clock() + self.settings.budget current = self.current = Pass(started=now_iso()) units = self.units(only, system) if dry_run: self.describe(units, system) return current isolation = core_probe.sandbox_argv([])[1] if isolation != core_probe.SANDBOX and not self.settings.allow_unsandboxed: # Fail closed: no core and no build runs outside the sandbox. current.result = "refused" current.detail = (f"no sandbox for third-party code ({isolation}). Install bubblewrap, " f"or pass --allow-unsandboxed") self.log(f"pass refused: {current.detail}") self.write_run_line(current) save_state(self.paths.state, self.state) return current for unit in units: for content in self.unit_contents(unit, system): self.store.hold(content, unit.emulator) done_units = 0 try: for unit in units: self.run_unit(unit, system) done_units += 1 save_state(self.paths.state, self.state) if commit and commit_every and done_units % commit_every == 0: self.commit_batch() except Pause as exc: current.result = "budget" if str(exc) == "budget" else "paused" current.detail = str(exc) self.log(f"pass paused: {exc}") except KeyboardInterrupt: current.result, current.detail = "stopped", "interrupted" finally: for unit in units: for content in self.unit_contents(unit, system): self.store.release(content, unit.emulator) self.prune_logs() self.write_run_line(current) save_state(self.paths.state, self.state) if commit: self.commit_batch() return current def run_unit(self, unit: Unit, system: str | None) -> None: emulator, profile = unit.emulator, unit.profile digest = profile_digest(profile) contents = self.unit_contents(unit, system) systems = [s for s in profile.get("systems") or () if not system or s == system] if unit.pending: for s in systems: self.pending(emulator, f"*/{s}/*", unit.pending, digest) return if not contents: for s in systems: self.pending(emulator, f"*/{s}/*", f"content: no test content listed for {s}", digest) return try: cores = self.unit_cores(emulator, profile, unit.buildbot_only) except Unavailable as exc: for s in systems: self.pending(emulator, f"*/{s}/*", str(exc), digest) self.log(f"{emulator}: {exc}") return for s in systems: self.state["checks"].setdefault(emulator, {}).pop(f"*/{s}/*", None) for core in cores: self.log(f"{emulator}: core {core.name} {core.sha256[:12]}") for content in contents: self.guard() self.run_unit_content(Unit(emulator, profile, core.name), core, content, digest) for content in contents: self.store.release(content, emulator) self.state["emulators"][emulator] = {"profile_digest": profile_digest( yaml.safe_load((self.paths.emulators / f"{emulator}.yml").read_text(encoding="utf-8"))), "at": now_iso()} # The campaign's own edits do not make its checks run again. new_digest = self.state["emulators"][emulator]["profile_digest"] for entry in self.state["checks"].get(emulator, {}).values(): if entry.get("profile_digest") == digest: entry["profile_digest"] = new_digest def unit_cores(self, emulator: str, profile: dict, buildbot_only: bool = False) -> list[Core]: names = self.cores.names(profile) if names: return [self.cores.nightly(name, profile) for name in names] if buildbot_only: raise Unavailable("standalone: no libretro core on the buildbot") return [self.cores.build(emulator, profile)] def describe(self, units: list[Unit], system: str | None) -> None: for unit in units: contents = self.unit_contents(unit, system) if unit.pending: print(f"{unit.emulator}: pending, {unit.pending}") elif not contents: print(f"{unit.emulator}: pending, no test content for " f"{', '.join(unit.profile.get('systems') or ())}") else: cores = sorted({str(c) for c in unit.profile.get("cores") or ()}) print(f"{unit.emulator}: cores {', '.join(cores)} on " f"{', '.join(c.name for c in contents)}") def counts(self) -> tuple[dict[str, int], dict[str, int], list[dict]]: counts = {DONE: 0, FAILED: 0, PENDING: 0} reasons: dict[str, int] = {} failed = [] for emulator, checks in sorted(self.state["checks"].items()): for key, entry in sorted(checks.items()): status = entry.get("status", PENDING) counts[status] = counts.get(status, 0) + 1 if status == PENDING: reason = reason_class(entry.get("reason", "")) reasons[reason] = reasons.get(reason, 0) + 1 elif status == FAILED: failed.append({"emulator": emulator, "check": key, "reason": entry.get("reason", "")}) return counts, reasons, failed def write_run_line(self, current: Pass) -> None: counts, reasons, failed = self.counts() line = mr.seal_line({ "schema_version": 1, "started_at": current.started, "ended_at": now_iso(), "result": current.result, "detail": current.detail, "counts": counts, "pending_reasons": reasons, "failed": failed, "records": current.records, "findings": dict(sorted(current.findings.items())), }, "started_at") path = self.paths.root / mr.records_path(mr.CAMPAIGN_RUNS) path.parent.mkdir(parents=True, exist_ok=True) with path.open("a", encoding="utf-8") as handle: handle.write(mr.line_text(line) + "\n") current.touched.add(mr.records_path(mr.CAMPAIGN_RUNS)) self.state["last_run"] = {k: line[k] for k in ("id", "started_at", "ended_at", "result", "detail")} def prune_logs(self) -> None: limit = time.time() - LOG_DAYS * 86400 for path in self.paths.logs.rglob("*.log") if self.paths.logs.is_dir() else (): if path.stat().st_mtime < limit: path.unlink() # -- commits (direct runs, autorun commits by its own rules) ------------------- def plan_commits(self, dirty_at_start: set[str]) -> list[tuple[str, list[str]]]: """(subject, paths) of each commit: the records, then one per profile.""" touched = sorted(self.current.touched if self.current else ()) records = [p for p in touched if p.startswith(f"{mr.RECORDS_DIR}/") and p not in dirty_at_start] plan = [] if records: plan.append(("chore: add campaign measurements", records)) for path in touched: if path.startswith("emulators/") and path not in dirty_at_start: subject = f"chore: apply tested fields to {Path(path).stem}" if len(subject) > 50: subject = "chore: apply tested profile fields" plan.append((subject, [path])) return plan def commit_batch(self, dry_run: bool = False) -> list[str]: plan = self.plan_commits(self.dirty_at_start) if dry_run or not plan: for subject, paths in plan: print(f"would commit {subject!r}: {', '.join(paths)}") return [] if gate(self.paths.root, [Path(p).stem for _, ps in plan for p in ps if p.startswith("emulators/")], self.host) != 0: self.log("gate failed, nothing committed") return [] made = [] for subject, paths in plan: problems = autorun.check_subject(subject) if problems: raise ValueError(f"{subject!r}: {', '.join(problems)}") made.append(autorun.commit_paths(self.paths.root, paths, autorun.CommitRule(subject, tuple(paths)))) if self.current: self.current.touched.clear() return made def reason_class(reason: str) -> str: """A pending reason without its detail, to count them.""" head = reason.split(":", 1)[0].strip() if ":" in reason else reason return head or "unknown" # -------------------------------------------------------------------------- # Gate # -------------------------------------------------------------------------- def gate(root: Path, emulators: Sequence[str], host: Host | None = None) -> int: """The measurement tests, the schemas, and verify plus an emulator pack for each profile the campaign changed.""" host = host or Host() python = sys.executable steps = [ [python, "-m", "unittest", "tests.test_measurements", "tests.test_campaign"], [python, "scripts/validate_schemas.py", "--source-only"], ] for emulator in sorted(set(emulators)): steps.append([python, "scripts/verify.py", "--emulator", emulator]) steps.append([python, "scripts/generate_pack.py", "--emulator", emulator, "--offline", "--output-dir", "tmp/campaign/packs"]) for argv in steps: proc = host.run(argv, cwd=root, check=False) if proc.returncode != 0: print(f"gate: {' '.join(argv)} exited {proc.returncode}", file=sys.stderr) return 1 shutil.rmtree(root / "tmp" / "campaign" / "packs", ignore_errors=True) return 0 def changed_profiles(root: Path) -> list[str]: return [Path(p).stem for p in autorun.changed_files(root) if p.startswith("emulators/") and p.endswith(".yml")] # -------------------------------------------------------------------------- # Reports # -------------------------------------------------------------------------- def status_report(campaign: Campaign) -> dict: counts, reasons, failed = campaign.counts() runs = mr.read_campaign(campaign.paths.root, mr.CAMPAIGN_RUNS) per = {} for emulator, checks in sorted(campaign.state["checks"].items()): per[emulator] = {key: {k: v for k, v in entry.items() if k in ("status", "reason", "record", "at")} for key, entry in sorted(checks.items())} return {"last_run": runs[-1] if runs else None, "counts": counts, "pending_reasons": reasons, "failed": failed, "emulators": per} def render_status(report: dict) -> str: lines = [] last = report["last_run"] if last: lines.append(f"last run {last['started_at']} to {last['ended_at']}: {last['result']}" + (f" ({last['detail']})" if last["detail"] else "")) else: lines.append("no run yet") counts = report["counts"] lines.append(f"{counts.get(DONE, 0)} done, {counts.get(FAILED, 0)} failed, " f"{counts.get(PENDING, 0)} pending") for reason, number in sorted(report["pending_reasons"].items(), key=lambda kv: -kv[1]): lines.append(f" pending {number:5} {reason}") for item in report["failed"]: lines.append(f" FAILED {item['emulator']} {item['check']}: {item['reason']}") for emulator, checks in report["emulators"].items(): ran = [k for k, v in checks.items() if v["status"] != PENDING] if ran: lines.append(f"{emulator}") for key in ran: entry = checks[key] lines.append(f" {entry['status']:7} {key}" + (f" {entry['record']}" if entry.get("record") else "") + (f" ({entry['reason']})" if entry.get("reason") else "")) return "\n".join(lines) def open_findings(root: Path, everything: bool = False) -> list[dict]: latest: dict[str, dict] = {} for line in mr.read_campaign(root, mr.FINDINGS): latest[line["key"]] = line return [line for key, line in sorted(latest.items()) if everything or line["status"] in ("open", "flagged")] # -------------------------------------------------------------------------- # Command line # -------------------------------------------------------------------------- def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) run_p = sub.add_parser("run", help="one pass: cores, content, checks, rules") run_p.add_argument("--only", default="", help="emulator profiles or core names, comma-separated") run_p.add_argument("--system", help="one system id") run_p.add_argument("--budget", type=parse_budget, default=6 * 3600.0, help="stop starting checks after this long (6h, 90m, 3600s)") run_p.add_argument("--max-load-wait", type=float, default=3600, help="seconds to wait for an idle host before pausing the pass") run_p.add_argument("--min-free-gb", type=float, default=30, help="pause below this free space") run_p.add_argument("--cache-cap-gb", type=float, default=20, help="content the campaign fetched is evicted past this size, oldest use first") run_p.add_argument("--romfetch", default="", help=f"romfetch command (default: ${ROMFETCH_ENV}, else romfetch on PATH)") run_p.add_argument("--offline", action="store_true", help="no buildbot, no romfetch, no build") run_p.add_argument("--allow-unsandboxed", action="store_true", help="run cores and builds without bubblewrap when it is missing (refused by default)") run_p.add_argument("--dry-run", action="store_true", help="print the plan, run nothing") run_p.add_argument("--commit", action="store_true", help="commit records, findings and each changed profile behind the gate") run_p.add_argument("--commit-every", type=int, default=0, metavar="N", help="with --commit, also commit after every N emulators") status_p = sub.add_parser("status", help="done, failed and pending checks, and why") status_p.add_argument("--json", action="store_true") findings_p = sub.add_parser("findings", help="profile claims the runs contradict, open for review") findings_p.add_argument("--all", action="store_true", help="applied and resolved too") findings_p.add_argument("--json", action="store_true") sub.add_parser("gate", help="tests, schemas, verify and pack of the changed profiles") timer_p = sub.add_parser("install-timer", help="write the autorun units, the campaign job's timer among them") timer_p.add_argument("--enable", action="store_true") args = parser.parse_args(argv) paths = Paths.default() if args.command == "gate": return gate(paths.root, changed_profiles(paths.root)) if args.command == "install-timer": command = [sys.executable, str(ROOT / "scripts" / "autorun.py"), "install-timers"] return subprocess.run(command + (["--enable"] if args.enable else []), check=False).returncode if args.command == "findings": lines = open_findings(paths.root, args.all) if args.json: print(json.dumps(lines, indent=2)) for line in [] if args.json else lines: print(f"{line['status']:8} {line['key']}: profile says {json.dumps(line['profile_says'])}, " f"run shows {json.dumps(line['run_shows'])}. {line['statement']} " f"[{', '.join(line['evidence'])}]") return 0 settings = Settings() if args.command == "run": settings = Settings(budget=args.budget, max_load_wait=args.max_load_wait, min_free_gb=args.min_free_gb, cache_cap_gb=args.cache_cap_gb, romfetch=args.romfetch, network=not args.offline, allow_unsandboxed=args.allow_unsandboxed) campaign = Campaign(paths, settings) if args.command == "status": report = status_report(campaign) print(json.dumps(report, indent=2) if args.json else render_status(report)) return 0 only = {item.strip() for item in args.only.split(",") if item.strip()} if args.dry_run: campaign.run(only, args.system, dry_run=True) return 0 campaign.dirty_at_start = set(autorun.changed_files(paths.root)) if args.commit else set() def stop(signum: int, frame: object) -> None: # A timer's TimeoutStartSec ends the pass like Ctrl-C: state saved, pass line written. raise KeyboardInterrupt signal.signal(signal.SIGTERM, stop) try: with host_lock(campaign.lock_path): current = campaign.run(only, args.system, commit_every=args.commit_every, commit=args.commit) except autorun.LockBusy as exc: print(f"campaign: deferred, {exc}", file=sys.stderr) return 75 counts, _, _ = campaign.counts() report = {"result": current.result, "detail": current.detail, "counts": counts, "records": len(current.records), "findings": current.findings} if current.result == "refused": # autorun raises an alert from this key of a job's report. report["alert"] = current.detail print(json.dumps(report)) if current.result == "refused": return EXIT_REFUSED return 75 if current.result == "paused" else 0 if __name__ == "__main__": from common import run_cli run_cli(main)