chore: merge the automatic test campaign

scripts/autorun.py

@@ -53,6 +53,7 @@ JOBS_FILE = REPO_ROOT / "automation" / "jobs.yml"
 AUTORUN_DIR = Path(".cache") / "autorun"
 DEFAULT_LOCK = Path.home() / ".local" / "state" / "retrobios" / "auto.lock"
 LOCK_ENV = "RETROBIOS_AUTO_LOCK"
+LOCK_HELD_ENV = "RETROBIOS_AUTO_LOCK_HELD"
 
 BRANCH = "autorun"
 REMOTE = "origin"
@@ -99,6 +100,10 @@ class Step:
     argv: tuple[str, ...]
     ok_codes: tuple[int, ...] = (0,)
     stdout: str = ""
+    # Variables passed through beside the allowlist (romfetch's settings).
+    env: tuple[str, ...] = ()
+    # GITHUB_TOKEN for a first-party script that calls the GitHub API.
+    github_token: bool = False
 
 
 @dataclass(frozen=True)
@@ -191,17 +196,61 @@ def check_subject(subject: str, body: str = "") -> list[str]:
     return problems(f"{subject}\n\n{body}" if body else subject)
 
 
-def _step(raw: object, where: str) -> Step:
+# The first-party scripts that call the GitHub API and may receive
+# GITHUB_TOKEN: run as `python <script>` or `python -m <module>`, and tracked
+# by git in this repository. Nothing else ever gets it, third-party code least.
+TOKEN_SCRIPTS = ("scripts/check_freshness.py", "scripts/refresh_stale.py")
+TOKEN_MODULE_PREFIX = "scripts.scraper."
+NO_TOKEN_JOBS = frozenset({"campaign"})
+
+
+def tracked(path: str, repo: Path = REPO_ROOT) -> bool:
+    proc = subprocess.run(["git", "-C", str(repo), "ls-files", "--error-unmatch", "--", path],
+                          capture_output=True, check=False)
+    return proc.returncode == 0
+
+
+def token_script(argv: Sequence[str], repo: Path = REPO_ROOT) -> str:
+    """The tracked first-party script a step runs, or '' when it runs anything else."""
+    if len(argv) < 2 or argv[0] != "python":
+        return ""
+    if argv[1] in TOKEN_SCRIPTS:
+        path = argv[1]
+    elif argv[1] == "-m" and len(argv) > 2 and argv[2].startswith(TOKEN_MODULE_PREFIX) \
+            and re.fullmatch(r"[a-z0-9_.]+", argv[2]):
+        path = argv[2].replace(".", "/") + ".py"
+    else:
+        return ""
+    return path if tracked(path, repo) else ""
+
+
+def _step(raw: object, where: str, job: str = "") -> Step:
     if isinstance(raw, list):
         return Step(argv=tuple(str(a) for a in raw))
     if isinstance(raw, dict) and isinstance(raw.get("run"), list):
-        unknown = set(raw) - {"run", "ok_codes", "stdout"}
+        unknown = set(raw) - {"run", "ok_codes", "stdout", "env", "github_token"}
         if unknown:
             raise ValueError(f"{where}: unknown step key {sorted(unknown)}")
+        names = tuple(str(n) for n in raw.get("env") or ())
+        secret = [n for n in names if is_secret(n)]
+        if secret:
+            raise ValueError(f"{where}: a step never receives {', '.join(secret)}")
+        argv = tuple(str(a) for a in raw["run"])
+        token = raw.get("github_token", False)
+        if token is not False:
+            if token is not True:
+                raise ValueError(f"{where}: github_token is true or absent")
+            if job in NO_TOKEN_JOBS:
+                raise ValueError(f"{where}: job {job} never receives GITHUB_TOKEN")
+            if not token_script(argv):
+                raise ValueError(f"{where}: github_token only goes to {', '.join(TOKEN_SCRIPTS)} "
+                                 f"or python -m {TOKEN_MODULE_PREFIX}*, tracked in this repository")
         return Step(
-            argv=tuple(str(a) for a in raw["run"]),
+            argv=argv,
             ok_codes=tuple(int(c) for c in raw.get("ok_codes", (0,))),
             stdout=str(raw.get("stdout") or ""),
+            env=names,
+            github_token=token is True,
         )
     raise ValueError(f"{where}: a step is a command list or a mapping with 'run'")
 
@@ -236,8 +285,8 @@ def parse_jobs(data: dict) -> dict[str, Job]:
             raise ValueError(f"job {name}: a job that commits runs in the worktree")
         jobs[name] = Job(
             name=name,
-            steps=tuple(_step(s, f"job {name}") for s in raw.get("steps") or ()),
-            gate=tuple(_step(s, f"job {name} gate") for s in raw.get("gate") or ()),
+            steps=tuple(_step(s, f"job {name}", name) for s in raw.get("steps") or ()),
+            gate=tuple(_step(s, f"job {name} gate", name) for s in raw.get("gate") or ()),
             twice=tuple(twice),
             commits=tuple(commits),
             schedule=str(raw.get("schedule") or ""),
@@ -438,9 +487,10 @@ def twice(
 # Git
 
 
-def git(cwd: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess:
+def git(cwd: Path, *args: str, check: bool = True,
+        env: dict[str, str] | None = None) -> subprocess.CompletedProcess:
     proc = subprocess.run(
-        ["git", "-C", str(cwd), *args], capture_output=True, text=True, check=False,
+        ["git", "-C", str(cwd), *args], capture_output=True, text=True, check=False, env=env,
     )
     if check and proc.returncode != 0:
         raise AutorunError(
@@ -565,7 +615,7 @@ def push_batch(worktree: Path, gate_base: str, regate: Callable[[], bool]) -> st
             raise AutorunError(BLOCKED, f"rebase onto {upstream} conflicts, batch kept")
     if git(worktree, "rev-parse", upstream).stdout.strip() != gate_base and not regate():
         raise AutorunError(FAILED, f"gate failed on the batch rebased onto {upstream}")
-    git(worktree, "push", "--quiet", REMOTE, f"HEAD:{TARGET}")
+    git(worktree, "push", "--quiet", REMOTE, f"HEAD:{TARGET}", env=token_env())
     return git(worktree, "rev-parse", "HEAD").stdout.strip()
 
 
@@ -577,6 +627,7 @@ def gh_json(args: list[str]) -> object:
     try:
         proc = subprocess.run(
             ["gh", *args], capture_output=True, text=True, timeout=120, check=False,
+            env=token_env(),
         )
     except (OSError, subprocess.TimeoutExpired):
         return None
@@ -699,14 +750,54 @@ def set_alert(state: dict, name: str, active: bool, detail: str, now: str) -> No
 # Running a job
 
 
-def job_env(job: Job) -> dict[str, str]:
-    env = os.environ.copy()
-    env["RETROBIOS_AUTO"] = "1"
-    if job.network:
-        sys.path.insert(0, str(REPO_ROOT / "scripts"))
-        from common import github_token
+# What a job step inherits. A step can run third-party code (a buildbot
+# core, a scraped page, a downloaded tool), so it gets the variables a
+# program needs to run and nothing it could leak: no token, no password.
+SAFE_ENV = frozenset({"PATH", "HOME", "LANG", "TZ", "TMPDIR"})
+SECRET_NAMES = frozenset({"GITHUB_TOKEN", "GH_TOKEN", "ROMSTATION_USER", "ROMSTATION_PASS"})
+SECRET_MARKERS = ("TOKEN", "SECRET", "PASS")
 
-        token = github_token()
+
+def is_secret(name: str) -> bool:
+    upper = name.upper()
+    return name in SECRET_NAMES or any(marker in upper for marker in SECRET_MARKERS)
+
+
+def safe_env(names: Sequence[str] = (), add: dict[str, str] | None = None,
+             source: dict[str, str] | None = None) -> dict[str, str]:
+    """The allowlisted environment, plus the variables a step names
+    explicitly. A secret is dropped even when named."""
+    source = dict(os.environ) if source is None else source
+    wanted = set(names)
+    env = {key: value for key, value in source.items()
+           if (key in SAFE_ENV or key.startswith("LC_") or key in wanted) and not is_secret(key)}
+    env.update({key: value for key, value in (add or {}).items() if not is_secret(key)})
+    return env
+
+
+def token_env() -> dict[str, str]:
+    """The environment of the phases that need GitHub: the push and gh."""
+    sys.path.insert(0, str(REPO_ROOT / "scripts"))
+    from common import github_token
+
+    env = dict(os.environ)
+    token = github_token()
+    if token:
+        env["GITHUB_TOKEN"] = token
+    return env
+
+
+def job_env(job: Job, step: Step | None = None) -> dict[str, str]:
+    """A step's environment: the allowlist, the runner's own markers and the
+    variables the step names. GITHUB_TOKEN only for a step that declares
+    github_token, which the registry allows on first-party scripts alone."""
+    add = {"RETROBIOS_AUTO": "1"}
+    if job.lock:
+        # A step that takes the host lock itself (campaign.py) runs under ours.
+        add[LOCK_HELD_ENV] = "1"
+    env = safe_env(step.env if step else (), add)
+    if step is not None and step.github_token and job.name not in NO_TOKEN_JOBS:
+        token = token_env().get("GITHUB_TOKEN")
         if token:
             env["GITHUB_TOKEN"] = token
     return env
@@ -744,7 +835,6 @@ class Runner:
         self.cwd = ctx.worktree if job.worktree else ctx.repo
         self.log = ctx.home / "logs" / job.name / f"{dt.date.today().isoformat()}.log"
         self.deadline = ctx.host.clock() + job.timeout
-        self.env: dict[str, str] = {}
         self.commits: list[str] = []
         self.retried: list[str] = []
         self.report: object = None
@@ -766,7 +856,7 @@ class Runner:
             left = self.deadline - self.ctx.host.clock()
             if left <= 0:
                 return Outcome(124, "job timeout reached")
-            return execute(argv, self.cwd, self.log, self.env, left, stdout)
+            return execute(argv, self.cwd, self.log, job_env(self.job, step), left, stdout)
 
         outcome = run_with_retry(once, step.ok_codes, self.wait_idle)
         if outcome.retried:
@@ -856,7 +946,6 @@ class Runner:
             refresh_breaker(self.state, ctx.gh, now_iso())
         if job.commits and self.state.get("breaker", {}).get("open"):
             return BLOCKED, "circuit breaker open: a pushed automation commit failed CI"
-        self.env = job_env(job)
         base = ""
         if job.worktree:
             ensure_worktree(ctx.repo, ctx.worktree)
@@ -940,6 +1029,11 @@ def run_job(ctx: Context, name: str) -> tuple[str, str]:
         else previous.get("consecutive_failures", 0) + 1,
     }
     state["jobs"][name] = record
+    # A job's report can raise an alert of its own: campaign.py refusing to
+    # run third-party code outside its sandbox.
+    alert = runner.report.get("alert") if isinstance(runner.report, dict) else None
+    if result != DEFERRED:
+        set_alert(state, f"job:{name}", bool(alert), str(alert or ""), now_iso())
     prune_logs(ctx.home / "logs")
     save_state(ctx.state_path, state)
     return result, detail

scripts/campaign_profiles.py

@@ -0,0 +1,161 @@
+"""Write a settled field into one entry of a profile, and nothing else.
+
+The profile is edited as text, so its comments, order and quoting stay as
+the maintainer wrote them. Every edit is checked by parsing the result: the
+edited entry must equal the old one with the new fields, and every other
+part of the profile must parse to what it did before. An edit that cannot
+hold both is refused (ProfileEditError), never written.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import yaml
+
+ITEM = re.compile(r"^(?P<indent>\s*)- ")
+KEY = re.compile(r"^(?P<indent>\s*)(?P<key>[A-Za-z_][A-Za-z0-9_]*):(?P<rest>.*)$")
+# Where a new key goes: after the first of these the entry has, else last.
+ANCHORS = {
+    "required": ("name",),
+    "hle_fallback": ("required", "name"),
+    "path": ("name",),
+    "aliases": ("path", "name"),
+}
+
+
+class ProfileEditError(ValueError):
+    """An edit the profile's layout does not allow to make safely."""
+
+
+def scalar(value: object) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if value is None:
+        return "null"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return json.dumps(value, ensure_ascii=False)
+    if isinstance(value, list):
+        return "[" + ", ".join(scalar(v) for v in value) + "]"
+    raise ProfileEditError(f"no scalar form for {value!r}")
+
+
+def item_spans(lines: list[str]) -> list[tuple[int, int, int]]:
+    """(first line, end line, item indent) of each item of the top-level files: list."""
+    try:
+        start = next(i for i, line in enumerate(lines) if line.rstrip() == "files:")
+    except StopIteration as exc:
+        raise ProfileEditError("no top-level files: list") from exc
+    spans: list[tuple[int, int, int]] = []
+    indent: int | None = None
+    current: int | None = None
+    end = len(lines)
+    for i in range(start + 1, len(lines)):
+        line = lines[i]
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        width = len(line) - len(line.lstrip(" "))
+        if width == 0 and not line.startswith("-"):
+            end = i
+            break
+        match = ITEM.match(line)
+        if match and (indent is None or width == indent):
+            indent = width
+            if current is not None:
+                spans.append((current, i, indent))
+            current = i
+    if current is not None:
+        spans.append((current, end, indent or 0))
+    # An item ends at its last line of content: the comments and blank lines
+    # before the next item belong to that item.
+    trimmed = []
+    for first, stop, width in spans:
+        last = stop
+        while last > first + 1 and (not lines[last - 1].strip() or lines[last - 1].strip().startswith("#")):
+            last -= 1
+        trimmed.append((first, last, width))
+    return trimmed
+
+
+def key_lines(lines: list[str], first: int, stop: int, key_indent: int) -> dict[str, tuple[int, int]]:
+    """key -> (line, end line) for each key of one item, its continuation lines included."""
+    keys: dict[str, tuple[int, int]] = {}
+    order: list[tuple[str, int]] = []
+    for i in range(first, stop):
+        # The item's first line carries the dash where its first key's indent is.
+        line = " " * key_indent + lines[i][key_indent:] if i == first else lines[i]
+        match = KEY.match(line)
+        if match and len(match["indent"]) == key_indent:
+            order.append((match["key"], i))
+    for position, (key, line) in enumerate(order):
+        end = order[position + 1][1] if position + 1 < len(order) else stop
+        while end > line + 1 and (not lines[end - 1].strip() or lines[end - 1].strip().startswith("#")):
+            end -= 1
+        keys[key] = (line, end)
+    return keys
+
+
+def _render(key: str, value: object, indent: int, block: bool = False) -> list[str]:
+    """One key as lines. A list keeps the block style it was written in."""
+    pad = " " * indent
+    if isinstance(value, dict):
+        return [f"{pad}{key}:"] + [f"{pad}  {k}: {scalar(v)}" for k, v in value.items()]
+    if block and isinstance(value, list):
+        return [f"{pad}{key}:"] + [f"{pad}  - {scalar(v)}" for v in value]
+    return [f"{pad}{key}: {scalar(value)}"]
+
+
+def set_fields(text: str, index: int, updates: dict[str, object]) -> str:
+    """The profile with these keys of files[index] set, the rest untouched."""
+    before = yaml.safe_load(text)
+    entries = before.get("files") or []
+    if not 0 <= index < len(entries):
+        raise ProfileEditError(f"no files[{index}]")
+    lines = text.splitlines()
+    spans = item_spans(lines)
+    if len(spans) != len(entries):
+        raise ProfileEditError(f"{len(spans)} items read as text, {len(entries)} parsed")
+    first, stop, item_indent = spans[index]
+    key_indent = item_indent + 2
+    for key, value in updates.items():
+        keys = key_lines(lines, first, stop, key_indent)
+        if key in keys:
+            start, end = keys[key]
+            rendered = _render(key, value, key_indent, block=lines[start].rstrip().endswith(":"))
+            if start == first:
+                raise ProfileEditError(f"files[{index}]: {key} is the item's first key")
+            lines[start:end] = rendered
+            stop += len(rendered) - (end - start)
+            continue
+        rendered = _render(key, value, key_indent)
+        anchor = next((keys[a][1] for a in ANCHORS.get(key, ()) if a in keys), None)
+        at = anchor if anchor is not None else stop
+        lines[at:at] = rendered
+        stop += len(rendered)
+    out = "\n".join(lines) + ("\n" if text.endswith("\n") else "")
+    after = yaml.safe_load(out)
+    expected = {**entries[index], **updates}
+    if after.get("files", [None] * (index + 1))[index] != expected:
+        raise ProfileEditError(f"files[{index}]: the edit does not parse to the intended entry")
+    rest_before = {k: v for k, v in before.items() if k != "files"}
+    rest_after = {k: v for k, v in after.items() if k != "files"}
+    others_before = [e for i, e in enumerate(entries) if i != index]
+    others_after = [e for i, e in enumerate(after.get("files") or []) if i != index]
+    if rest_before != rest_after or others_before != others_after:
+        raise ProfileEditError(f"files[{index}]: the edit changes another part of the profile")
+    return out
+
+
+def write_fields(path: Path, index: int, updates: dict[str, object]) -> bool:
+    """Apply the edit to the file. False when it changes nothing."""
+    text = path.read_text(encoding="utf-8")
+    out = set_fields(text, index, updates)
+    if out == text:
+        return False
+    path.write_text(out, encoding="utf-8")
+    return True

scripts/campaign_rules.py

@@ -0,0 +1,203 @@
+"""The campaign's fixed rule table: which observable settles which profile field.
+
+Each rule is a pure function of a core probe record (or what was read from
+one) and a profile entry. It returns verdicts. A verdict with `apply` set is
+settled by the table and written into the profile with the record ids as
+`evidence:`. Any other verdict goes to the review queue
+(measurements/findings.jsonl).
+
+  observable                                        field          settled
+  boots to the game with no file                    hle_fallback   true
+  no game screen without it, the game with it       required       true
+                                                    hle_fallback   false
+  opens the staged file under another path          path           that path
+  opens the staged file under another name          aliases        name added
+  a staged file the content should use, never read  -              flagged only
+
+"Boots to the game" is read from the sampled frames: the run without the
+file shows at least MIN_SHARED_SCREENS distinct screens that the run with
+the file also shows, single-colour frames excluded. "No game screen" is a
+run that did not load, or whose every sampled frame is a single colour. A
+run that shows screens of its own that the reference never shows settles
+nothing.
+
+Standard library only.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+
+MIN_SHARED_SCREENS = 2
+BOOTS = "boots"
+FAILS = "fails"
+INCONCLUSIVE = "inconclusive"
+
+# A field an entry leaves out reads as this value.
+FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": []}
+SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases")
+
+
+@dataclass(frozen=True)
+class Boot:
+    """What the runs without the file showed, against the runs with it."""
+
+    status: str
+    detail: str
+    shared: int = 0
+
+
+@dataclass(frozen=True)
+class Verdict:
+    rule: str
+    field: str
+    value: object
+    statement: str
+    evidence: tuple[str, ...]
+    apply: bool = False
+
+
+@dataclass(frozen=True)
+class Opened:
+    """Where a run found the staged file opened."""
+
+    dest: str
+    record_id: str
+    extra: dict = field(default_factory=dict)
+
+
+def screens(repetition: dict) -> set[str]:
+    """The distinct sampled frames of a repetition that are not one colour."""
+    observables = repetition.get("observables") or {}
+    uniform = set(observables.get("uniform_frames") or ())
+    return {digest for _, digest in observables.get("frame_hashes") or () if digest not in uniform}
+
+
+def variant(record: dict, label: str) -> dict:
+    for item in record.get("variants") or ():
+        if item.get("label") == label:
+            return item
+    raise KeyError(f"record {record.get('id')}: no variant {label!r}")
+
+
+def boot_outcome(record: dict, absent_label: str, present_label: str) -> Boot:
+    """Whether the core reached the game without the file.
+
+    Every repetition must agree: one that boots and one that does not leave
+    the question open.
+    """
+    present = variant(record, present_label)["repetitions"]
+    if any(not rep.get("loaded") for rep in present):
+        return Boot(INCONCLUSIVE, "with the file, the core did not load the content")
+    reference: set[str] = set()
+    for rep in present:
+        shown = screens(rep)
+        if not shown:
+            return Boot(INCONCLUSIVE, "with the file, the core showed no game screen")
+        reference |= shown
+    outcomes: list[str] = []
+    shared_counts: list[int] = []
+    for rep in variant(record, absent_label)["repetitions"]:
+        if not rep.get("loaded"):
+            outcomes.append(FAILS)
+            continue
+        shown = screens(rep)
+        if not shown:
+            outcomes.append(FAILS)
+            continue
+        shared = len(shown & reference)
+        shared_counts.append(shared)
+        outcomes.append(BOOTS if shared >= MIN_SHARED_SCREENS else INCONCLUSIVE)
+    shared = min(shared_counts) if shared_counts else 0
+    if all(o == BOOTS for o in outcomes):
+        return Boot(BOOTS, f"without the file, every run showed {shared} or more screens the "
+                           f"run with the file shows", shared)
+    if all(o == FAILS for o in outcomes):
+        return Boot(FAILS, "without the file, no run loaded the content or showed more than "
+                           "one colour, while the run with the file reached the game")
+    return Boot(INCONCLUSIVE, "without the file, the runs showed screens the run with the file "
+                              "never shows, or disagreed with each other", shared)
+
+
+def current(entry: dict, name: str) -> object:
+    if name in entry:
+        return entry[name]
+    return FIELD_DEFAULTS.get(name)
+
+
+def agrees(entry: dict, verdict: Verdict) -> bool:
+    """True when the profile already says what the run showed."""
+    if verdict.field == "aliases":
+        return set(verdict.value or ()) <= set(entry.get("aliases") or ())
+    return current(entry, verdict.field) == verdict.value
+
+
+def rule_bios_need(boot: Boot, entry: dict, record_id: str) -> list[Verdict]:
+    """Boot with no file against boot with the declared file."""
+    name = entry["name"]
+    evidence = (record_id,)
+    if boot.status == BOOTS:
+        out = [Verdict("boots_without_file", "hle_fallback", True,
+                       f"the core reaches the game without {name}: {boot.detail}", evidence, True)]
+        if entry.get("required") is True:
+            out.append(Verdict(
+                "boots_without_file", "required", False,
+                f"the profile says {name} is required, and the core reaches the game "
+                f"without it. The table settles hle_fallback from this run, not required",
+                evidence))
+        return out
+    if boot.status == FAILS:
+        return [
+            Verdict("fails_without_file", "required", True,
+                    f"the core reaches the game only with {name}: {boot.detail}", evidence, True),
+            Verdict("fails_without_file", "hle_fallback", False,
+                    f"no stand-in for {name}: {boot.detail}", evidence, True),
+        ]
+    return [Verdict("inconclusive", "hle_fallback", None,
+                    f"the runs with and without {name} settle nothing: {boot.detail}", evidence)]
+
+
+def rule_path(entry: dict, declared: str, opened: Opened) -> Verdict | None:
+    """The core opened the file at another place under the system directory."""
+    if opened.dest == declared:
+        return None
+    return Verdict("opened_under_other_path", "path", opened.dest,
+                   f"the core opened {entry['name']} as {opened.dest}, the profile declares "
+                   f"{declared}", (opened.record_id,), True)
+
+
+def rule_alias(entry: dict, opened: Opened) -> Verdict | None:
+    """The core opened the file under a name the entry does not declare."""
+    name = opened.dest.rsplit("/", 1)[-1]
+    known = [entry["name"], *(entry.get("aliases") or ())]
+    if name in known:
+        return None
+    return Verdict("opened_other_name", "aliases", [*(entry.get("aliases") or ()), name],
+                   f"the core opened {entry['name']} under the name {name}",
+                   (opened.record_id,), True)
+
+
+def rule_never_opened(entry: dict, record_ids: Sequence[str], content: str) -> Verdict:
+    return Verdict("never_opened", "opened", False,
+                   f"{entry['name']} was staged and the core never opened it while running "
+                   f"{content}", tuple(record_ids))
+
+
+def rule_variants(status: str, labels: Iterable[str], entry: dict, record_id: str) -> Verdict | None:
+    """Copies of one file that change what the core shows: a pack choice to review."""
+    if status != "differs":
+        return None
+    return Verdict("variant_changes_output", "variants", sorted(labels),
+                   f"the copies of {entry['name']} change what the core shows",
+                   (record_id,))
+
+
+def merge_evidence(existing: dict | None, verdict: Verdict) -> dict:
+    """The entry's evidence: map with the verdict's record ids added under its field."""
+    out = {key: list(value) for key, value in (existing or {}).items()}
+    ids = out.setdefault(verdict.field, [])
+    for rid in verdict.evidence:
+        if rid not in ids:
+            ids.append(rid)
+    return {key: out[key] for key in sorted(out)}

scripts/core_probe.py

@@ -18,6 +18,7 @@ reproduce it (core build, frontend commit, host, inputs by hash).
 from __future__ import annotations
 
 import argparse
+import contextlib
 import ctypes
 import datetime as dt
 import hashlib
@@ -26,6 +27,7 @@ import os
 import platform
 import re
 import shutil
+import signal
 import statistics
 import struct
 import subprocess
@@ -269,6 +271,14 @@ def render_printf(fmt: bytes, args: Sequence[int]) -> str:
     return "".join(out).rstrip("\n")
 
 
+def uniform_frame(frame: bytes, pixel_bytes: int) -> bool:
+    """True for a frame of one colour, an empty one included: a screen that
+    shows nothing of the game."""
+    if len(frame) <= pixel_bytes:
+        return True
+    return frame == frame[:pixel_bytes] * (len(frame) // pixel_bytes)
+
+
 def png_bytes(width: int, height: int, rgb: bytes) -> bytes:
     def chunk(tag: bytes, body: bytes) -> bytes:
         return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body))
@@ -359,6 +369,7 @@ class Frontend:
         self.video = hashlib.sha1()
         self.audio = hashlib.sha1()
         self.frame_hashes: list[list] = []
+        self.uniform: set[str] = set()
         self.logs: list[dict] = []
         self.presses: list[tuple[int, int, int]] = [tuple(p) for p in spec["presses"]]
         self.unanswered: dict[int, int] = {}
@@ -433,7 +444,10 @@ class Frontend:
         self.video.update(struct.pack("<II", *self.last_size) + hashlib.sha1(self.last_frame).digest())
         every = self.spec["frame_hash_every"]
         if every and self.frame % every == 0:
-            self.frame_hashes.append([self.frame, hashlib.sha1(self.last_frame).hexdigest()])
+            digest = hashlib.sha1(self.last_frame).hexdigest()
+            self.frame_hashes.append([self.frame, digest])
+            if digest not in self.uniform and uniform_frame(self.last_frame, PIXEL_BYTES[self.pixel_format]):
+                self.uniform.add(digest)
             png_dir = self.spec.get("png_dir")
             if png_dir and self.last_frame:
                 rgb = to_rgb(self.pixel_format, *self.last_size, self.last_frame)
@@ -458,6 +472,9 @@ class Frontend:
 
 def run_worker(spec: dict) -> dict:
     """Load the core in this process and run it. Returns the measured result."""
+    # Nothing of the host's environment reaches the core: only what --env gave.
+    os.environ.clear()
+    os.environ.update(spec["env"])
     core = ctypes.CDLL(spec["core"])
     front = Frontend(spec)
     for name, fn in zip(("retro_set_environment", "retro_set_video_refresh", "retro_set_audio_sample",
@@ -514,6 +531,7 @@ def run_worker(spec: dict) -> dict:
         observables[f"state:{name}"] = state_vars.get(name)
     if spec["frame_hash_every"]:
         observables["frame_hashes"] = front.frame_hashes
+        observables["uniform_frames"] = sorted(front.uniform)
     core.retro_unload_game()
     core.retro_deinit()
     return {
@@ -584,7 +602,7 @@ def frontend_identity() -> dict:
 
 def core_identity(args: argparse.Namespace) -> dict:
     core = Path(args.core)
-    return {
+    identity = {
         "file": core.name,
         "sha256": file_digest(core, "sha256"),
         "repo": args.core_repo,
@@ -592,6 +610,9 @@ def core_identity(args: argparse.Namespace) -> dict:
         "build_command": args.build_command,
         "compiler": elf_comment(core) or None,
     }
+    if args.core_artifact:
+        identity["artifact"] = args.core_artifact
+    return identity
 
 
 # --------------------------------------------------------------------------
@@ -630,6 +651,84 @@ def host_paths_hidden(text: str, places: Sequence[tuple[str, str]]) -> str:
     return text
 
 
+_BWRAP_WORKS: dict[str, bool] = {}
+
+
+SANDBOX = "bwrap"
+# What a core or a build sees of the host, read-only: the system's programs
+# and libraries, and the parts of /etc the dynamic loader, fonts, TLS, the
+# clock and user lookups read. Nothing else of the tree exists inside.
+SANDBOX_SYSTEM = ("/usr", "/lib", "/lib32", "/lib64", "/libx32", "/bin", "/sbin")
+SANDBOX_ETC = ("/etc/ld.so.cache", "/etc/ld.so.conf", "/etc/ld.so.conf.d", "/etc/fonts",
+               "/etc/ssl", "/etc/ca-certificates", "/etc/localtime", "/etc/nsswitch.conf",
+               "/etc/passwd", "/etc/group", "/etc/alternatives")
+
+
+def _bwrap_base(bwrap: str, network: bool) -> list[str]:
+    argv = [bwrap, "--unshare-all", "--die-with-parent", "--new-session"]
+    if network:
+        argv.append("--share-net")
+    for path in SANDBOX_SYSTEM:
+        if os.path.islink(path):
+            # usrmerge: /lib -> usr/lib is a link, made again inside.
+            argv += ["--symlink", os.readlink(path), path]
+        elif os.path.isdir(path):
+            argv += ["--ro-bind", path, path]
+    for path in SANDBOX_ETC:
+        argv += ["--ro-bind-try", path, path]
+    argv += ["--ro-bind-try", "/sys/devices/system/cpu", "/sys/devices/system/cpu",
+             "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/dev/shm", "--tmpfs", "/tmp",
+             "--tmpfs", "/mnt", "--tmpfs", str(Path.home())]
+    return argv
+
+
+def sandbox_argv(writable: Sequence[Path], readonly: Sequence[Path] = (),
+                 network: bool = False) -> tuple[list[str], str]:
+    """The bubblewrap prefix a third-party program runs under, and its name.
+
+    An allowlist: the system's programs and libraries and a few files of
+    /etc read-only, an empty home, empty /tmp and /mnt, the `readonly` paths
+    (the core, the content, the staged system directory) read-only, and only
+    `writable` writable. Every namespace is unshared, the network too unless
+    `network` asks for it. Without a working bwrap the returned prefix is
+    empty and the name says why, so the caller can refuse to run.
+    """
+    bwrap = shutil.which("bwrap")
+    if not bwrap:
+        return [], "environment only: bwrap not found"
+    if bwrap not in _BWRAP_WORKS:
+        probe_run = subprocess.run(_bwrap_base(bwrap, False) + ["/bin/true"], capture_output=True,
+                                   check=False, env={})
+        _BWRAP_WORKS[bwrap] = probe_run.returncode == 0
+    if not _BWRAP_WORKS[bwrap]:
+        return [], "environment only: bwrap cannot create a sandbox here"
+    argv = _bwrap_base(bwrap, network)
+    for path in readonly:
+        resolved = Path(path).resolve()
+        argv += ["--ro-bind", str(resolved), str(resolved)]
+    for path in writable:
+        resolved = Path(path).resolve()
+        resolved.mkdir(parents=True, exist_ok=True)
+        argv += ["--bind", str(resolved), str(resolved)]
+    return argv, SANDBOX
+
+
+def run_group(command: list[str], timeout: float, env: dict[str, str]) -> subprocess.CompletedProcess:
+    """Run the worker in a process group of its own and kill the whole group
+    when it outlives the timeout. Killing strace alone left a hung core
+    spinning, detached from anything that would ever stop it."""
+    with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+                          env=env, start_new_session=True) as proc:
+        try:
+            out, err = proc.communicate(timeout=timeout)
+        except BaseException:
+            with contextlib.suppress(ProcessLookupError):
+                os.killpg(proc.pid, signal.SIGKILL)
+            proc.communicate()
+            raise
+    return subprocess.CompletedProcess(command, proc.returncode, out, err)
+
+
 def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None,
              png_dir: Path | None) -> dict:
     save_dir = scratch / "save"
@@ -648,6 +747,7 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou
         "joypad_ports": args.joypad_ports,
         "png_dir": str(png_dir) if png_dir else None,
         "state_out": str(state_out) if state_out else None,
+        "env": dict(args.env),
     }
     spec_path, out_path, trace_path = scratch / "spec.json", scratch / "result.json", scratch / "strace.txt"
     spec_path.write_text(json.dumps(spec), encoding="utf-8")
@@ -655,9 +755,16 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou
     strace = shutil.which("strace") if not args.no_trace else None
     if strace:
         command = [strace, "-f", "-qq", "-s", "4096", "-e", "trace=openat,open", "-o", str(trace_path)] + command
-    env = {**os.environ, **dict(args.env)}
-    proc = subprocess.run(command, check=False, capture_output=True, text=True,
-                          timeout=args.timeout, env=env)
+    writable = [scratch] + [p for p in (png_dir, state_out.parent if state_out else None) if p]
+    readonly = [Path(__file__), Path(args.core), system_dir]
+    if args.content:
+        readonly.append(Path(args.content).resolve().parent)
+    readonly += [Path(value) for _, value in args.env if Path(value).is_absolute() and Path(value).exists()]
+    prefix, isolation = sandbox_argv(writable, readonly)
+    if args.require_sandbox and isolation != SANDBOX:
+        raise ProbeError(f"refused to run the core outside a sandbox: {isolation}")
+    # The core sees the variables it was given and nothing of the host's.
+    proc = run_group(prefix + command, args.timeout, dict(args.env))
     if proc.returncode != 0 or not out_path.exists():
         raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}")
     result = json.loads(out_path.read_text(encoding="utf-8"))
@@ -681,6 +788,7 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou
     if result.get("wall_s") and result.get("frames"):
         result["ms_per_frame"] = result["wall_s"] * 1000 / result["frames"]
         result["fps"] = result["frames"] / result["wall_s"]
+    result["isolation"] = isolation
     return result
 
 
@@ -769,6 +877,8 @@ def build_parser() -> argparse.ArgumentParser:
     common.add_argument("--core-repo", help="source repository URL of the core build")
     common.add_argument("--core-commit", help="commit the core was built from")
     common.add_argument("--build-command", help="command and flags that built the core")
+    common.add_argument("--core-artifact", metavar="URL",
+                        help="where the .so was downloaded from, such as a buildbot nightly")
     common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
                         help="append the record as one line of measurements/<emulator>.jsonl "
                              "in the checkout DIR (default: this one)")
@@ -786,6 +896,8 @@ def build_parser() -> argparse.ArgumentParser:
     common.add_argument("--work-dir", default=str(DEFAULT_WORK_DIR), help="scratch directory")
     common.add_argument("--timeout", type=int, default=1800, help="seconds per repetition")
     common.add_argument("--no-trace", action="store_true", help="do not run the worker under strace")
+    common.add_argument("--require-sandbox", action="store_true",
+                        help="refuse to run the core unless bubblewrap can isolate it")
 
     sub.add_parser("run", parents=[common], help="one execution")
     bench = sub.add_parser("bench", parents=[common], help="timed repetitions")
@@ -806,6 +918,20 @@ def build_parser() -> argparse.ArgumentParser:
     return parser
 
 
+def not_a_checkout_root(directory: Path) -> str | None:
+    """Why --record DIR is refused, or None. DIR is the checkout, never its
+    measurements/ folder: that wrote measurements/measurements/<emulator>.jsonl."""
+    form = "--record names the root of a git checkout (the folder holding measurements/), e.g. --record ."
+    if not (directory / ".git").exists():
+        return f"--record {directory}: not a git checkout root. {form}"
+    proc = subprocess.run(["git", "-C", str(directory), "rev-parse", "--show-toplevel"],
+                          capture_output=True, text=True, check=False)
+    top = proc.stdout.strip()
+    if proc.returncode != 0 or not top or Path(top).resolve() != directory.resolve():
+        return f"--record {directory}: not a git checkout root (its root is {top or 'unknown'}). {form}"
+    return None
+
+
 def validate_args(args: argparse.Namespace) -> list[str]:
     problems: list[str] = []
     if not Path(args.core).is_file():
@@ -817,9 +943,16 @@ def validate_args(args: argparse.Namespace) -> list[str]:
             problems.append(f"--system-file {dest}={src}: not a file")
     if args.frames < 1 or args.warmup < 0:
         problems.append("--frames must be >= 1 and --warmup >= 0")
+    if args.require_sandbox:
+        isolation = sandbox_argv([])[1]
+        if isolation != SANDBOX:
+            problems.append(f"--require-sandbox: {isolation}")
     if args.png_dir and not args.frame_hash_every:
         problems.append("--png-dir needs --frame-hash-every")
     if args.record:
+        wrong = not_a_checkout_root(Path(args.record))
+        if wrong:
+            problems.append(wrong)
         if not args.emulator and Path(args.core).is_file():
             try:
                 default_emulator(Path(args.core), None)
@@ -841,7 +974,7 @@ def validate_args(args: argparse.Namespace) -> list[str]:
                 problems.append(f"--vary {dest}={src}: not a file")
         names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var}
         if args.frame_hash_every:
-            names.add("frame_hashes")
+            names.update(("frame_hashes", "uniform_frames"))
         if args.observable not in names:
             problems.append(f"--observable {args.observable}: not one of {', '.join(sorted(names))}")
     return problems
@@ -936,8 +1069,9 @@ def default_subject(record: dict) -> str:
     return f"{record['kind']}-{slugify(content, 30)}-{digest}"
 
 
-def write_record(directory: Path, full: dict) -> Path:
-    """Append the record, with its id, to its emulator's .jsonl."""
+def write_record(directory: Path, full: dict) -> tuple[Path, dict]:
+    """Append the record, with its id, to its emulator's .jsonl. Returns the
+    file and the record as written."""
     from measurement_records import line_text, parse_lines, records_path, seal  # noqa: PLC0415
 
     sealed = seal(full)
@@ -948,7 +1082,7 @@ def write_record(directory: Path, full: dict) -> Path:
     lines.parent.mkdir(parents=True, exist_ok=True)
     with lines.open("a", encoding="utf-8") as handle:
         handle.write(line_text(sealed) + "\n")
-    return lines
+    return lines, sealed
 
 
 def probe(args: argparse.Namespace) -> int:
@@ -999,7 +1133,7 @@ def probe(args: argparse.Namespace) -> int:
             "repetitions": repetitions,
             "presses": [{"frame": f, "button": b, "length": n} for f, b, n in args.press],
             "observables": list(BASE_OBSERVABLES) + [f"state:{v}" for v in args.state_var]
-                           + (["frame_hashes"] if args.frame_hash_every else []),
+                           + (["frame_hashes", "uniform_frames"] if args.frame_hash_every else []),
             "frame_hash_every": args.frame_hash_every,
             "joypad_ports": args.joypad_ports,
         },
@@ -1025,7 +1159,7 @@ def probe(args: argparse.Namespace) -> int:
     record["subject"] = args.subject or default_subject(record)
     if args.record:
         try:
-            lines = write_record(Path(args.record), record)
+            lines, record = write_record(Path(args.record), record)
         except ProbeError as exc:
             print(f"ERROR: {exc}", file=sys.stderr)
             return 2
@@ -1050,7 +1184,10 @@ def main(argv: Sequence[str] | None = None) -> int:
 
 
 if __name__ == "__main__":
-    # The worker runs under -I, which leaves the script directory off sys.path.
+    if sys.argv[1:2] == ["_worker"]:
+        # The worker runs in the sandbox, where this file is all of the
+        # repository there is: it imports nothing beside the standard library.
+        sys.exit(main())
     sys.path.insert(0, str(Path(__file__).resolve().parent))
     from common import run_cli
 

scripts/measurement_records.py

@@ -81,6 +81,47 @@ def records_path(emulator: str) -> str:
     return f"{RECORDS_DIR}/{emulator}.jsonl"
 
 
+# The campaign's own files beside the records: append-only like them, one
+# line per finding or per pass, each line hashed to its id. No profile takes
+# these names (tests/test_campaign.py checks it).
+FINDINGS = "findings"
+CAMPAIGN_RUNS = "campaign-runs"
+CAMPAIGN_FILES = {
+    FINDINGS: ("found_at", "campaign-finding.schema.json"),
+    CAMPAIGN_RUNS: ("started_at", "campaign-run.schema.json"),
+}
+
+
+def line_id(line: dict, date_key: str) -> str:
+    body = {key: value for key, value in line.items() if key != "id"}
+    return f"{line[date_key][:10]}-{sha256_hex(canonical(body))[:12]}"
+
+
+def seal_line(line: dict, date_key: str) -> dict:
+    return {**line, "id": line_id(line, date_key)}
+
+
+def read_campaign(root: Path, name: str) -> list[dict]:
+    """The lines of one campaign file, each checked against its id."""
+    path = root / records_path(name)
+    if not path.is_file():
+        return []
+    date_key = CAMPAIGN_FILES[name][0]
+    lines = parse_lines(path.read_text(encoding="utf-8"), records_path(name))
+    problems = []
+    seen: set[str] = set()
+    for number, line in enumerate(lines, 1):
+        rid = str(line.get("id", ""))
+        if not ID_PATTERN.match(rid) or date_key not in line or line_id(line, date_key) != rid:
+            problems.append(f"{records_path(name)}:{number}: id {rid!r} does not hash the line")
+        if rid in seen:
+            problems.append(f"{records_path(name)}:{number}: id {rid} appears twice")
+        seen.add(rid)
+    if problems:
+        raise MeasurementsError("\n  ".join(problems))
+    return lines
+
+
 IDENTITY = ("id", "emulator", "subject", "supersedes", "migrated_from_version")
 
 
@@ -177,6 +218,8 @@ def read_records(root: Path) -> list[Stored]:
         if not path.endswith(".jsonl") or "/" in emulator:
             problems.append(f"{path}: {RECORDS_DIR}/ holds <emulator>.jsonl files only")
             continue
+        if emulator in CAMPAIGN_FILES:
+            continue
         seen: set[str] = set()
         try:
             records = parse_lines(files[path].read_text(encoding="utf-8"), path)

scripts/sitemeasurements.py

@@ -140,6 +140,105 @@ def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
     }
 
 
+def campaign_summary(root: Path, stored: list[mr.Stored], source: dict | None) -> dict:
+    """The last campaign pass and the findings still waiting for review.
+
+    A finding's history is its lines under one key: the latest line is its
+    state, and only an open or flagged one is listed.
+    """
+    runs = mr.read_campaign(root, mr.CAMPAIGN_RUNS)
+    latest: dict[str, dict] = {}
+    applied = 0
+    for line in mr.read_campaign(root, mr.FINDINGS):
+        latest[line["key"]] = line
+        applied += line["status"] == "applied"
+    where = {s.record["id"]: s for s in stored}
+    links = {}
+    for line in latest.values():
+        for rid in line["evidence"]:
+            found = where.get(rid)
+            if found and source and source.get("commit"):
+                links[rid] = permalink(source["repo"], source["commit"],
+                                       mr.records_path(found.emulator), found.line)
+    open_items = [{**line, "links": [links.get(rid) for rid in line["evidence"]]}
+                  for _, line in sorted(latest.items()) if line["status"] in ("open", "flagged")]
+    return {"last_run": runs[-1] if runs else None, "passes": len(runs), "applied": applied,
+            "open": open_items}
+
+
+# Characters a campaign value could use to leave its place on the page: HTML
+# and MDX syntax ({ } opens an expression), code spans, emphasis and links.
+_UNSAFE = {"&": "&amp;", "<": "&lt;", ">": "&gt;", "{": "&#123;", "}": "&#125;", "`": "&#96;",
+           "[": "&#91;", "]": "&#93;", "*": "&#42;", "_": "&#95;", "\\": "&#92;", "|": "&#124;"}
+
+
+def _plain(value: object) -> str:
+    """Text a run produced (core output, a romfetch error, a profile name),
+    shown as text: every character that is markup on the page is a
+    character reference."""
+    text = " ".join(str(value if value is not None else "").split())
+    return "".join(_UNSAFE.get(char, char) for char in text)
+
+
+def _sentence(value: object) -> str:
+    """A finding's statement as a sentence of the page: capital first."""
+    text = _plain(value)
+    return text[:1].upper() + text[1:]
+
+
+def _code(value: object) -> str:
+    """A value as code that cannot close its element or open an expression."""
+    return f"<code>{_plain(value)}</code>"
+
+
+def _value(value: object) -> str:
+    return _code(json.dumps(value, ensure_ascii=False)) if value is not None else "nothing"
+
+
+def campaign_section(summary: dict | None) -> list[str]:
+    """The campaign's last pass and its open findings, for the explorer page."""
+    if not summary or not summary.get("last_run"):
+        return []
+    last = summary["last_run"]
+    counts = last["counts"]
+    lines = [
+        "## Campaign",
+        "",
+        f"Tests run by themselves every night on an idle computer ([how]({METHOD_PAGE})). "
+        f"Last pass: {_plain(last['ended_at'][:10])}, {_plain(last['result'])}"
+        + (f" ({_plain(last['detail'])})" if last.get("detail") else "") + ". "
+        f"{int(counts['done'])} tests done, {int(counts['failed'])} failed, {int(counts['pending'])} waiting. "
+        f"{summary['applied']} profile fields were set from a run.",
+        "",
+    ]
+    if last["pending_reasons"]:
+        lines += ["| Waiting for | Tests |", "|-------------|------:|"]
+        lines += [f"| {_plain(reason)} | {int(number)} |"
+                  for reason, number in sorted(last["pending_reasons"].items(), key=lambda kv: (-kv[1], kv[0]))]
+        lines.append("")
+    if last["failed"]:
+        lines += ["Failed tests:", ""]
+        lines += [f"- {_plain(item['emulator'])}, {_code(item['check'])}: {_plain(item['reason'])}"
+                  for item in last["failed"]]
+        lines.append("")
+    lines += ["### Open findings", ""]
+    if not summary["open"]:
+        lines += ["None: every finding was settled by a rule or resolved by a later run.", ""]
+        return lines
+    lines += ["A finding is what a run showed that no rule settles. It waits for someone to "
+              "read the emulator's code.", ""]
+    for item in summary["open"]:
+        records = ", ".join(f"[{_plain(rid)}]({link})" if link else _code(rid)
+                            for rid, link in zip(item["evidence"], item["links"]))
+        lines += [
+            f"- **{_plain(item['emulator'])}**, {_code(item['file'])}, {_code(item['field'])} "
+            f"({_plain(item['status'])}): the profile says {_value(item['profile_says'])}, the run "
+            f"shows {_value(item['run_shows'])}. {_sentence(item['statement'])}. Records: {records}.",
+            "",
+        ]
+    return lines
+
+
 # --------------------------------------------------------------------------
 # SQLite
 # --------------------------------------------------------------------------
@@ -360,6 +459,7 @@ def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulato
     source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)}
     profiles = load_profiles(emulators_dir)
     index = build_index(stored, source, profiles, system_names)
+    index["campaign"] = campaign_summary(root, stored, source)
     paths = [public / INDEX_PATH]
     (public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
     write_if_changed(str(public / INDEX_PATH),
@@ -485,6 +585,7 @@ def generate_explorer_page(index: dict) -> str:
         "",
         "<MeasurementsExplorer />",
         "",
+        *campaign_section(index.get("campaign")),
     ])
 
 

scripts/validate_schemas.py

@@ -601,6 +601,8 @@ def _measurement_errors(root: Path | None = None) -> list[str]:
     root = root or ROOT
     validator = _validator("core-probe-record.schema.json")
     out: list[str] = []
+    campaign = {mr.records_path(name): _validator(schema)
+                for name, (_, schema) in mr.CAMPAIGN_FILES.items()}
     for path, file in sorted(mr.tree_files(root).items()):
         if not path.endswith(".jsonl"):
             continue
@@ -610,10 +612,12 @@ def _measurement_errors(root: Path | None = None) -> list[str]:
             except json.JSONDecodeError as exc:
                 out.append(f"{path}:{number}: {exc}")
                 continue
-            out.extend(_errors(validator, record, f"{path}:{number}"))
+            out.extend(_errors(campaign.get(path, validator), record, f"{path}:{number}"))
     if not out:
         try:
             mr.read_records(root)
+            for name in mr.CAMPAIGN_FILES:
+                mr.read_campaign(root, name)
         except mr.MeasurementsError as exc:
             out.append(str(exc))
     return out