feat: add the real-condition campaign runner

campaign.py takes buildbot nightlies (or a build of the pinned commit), the test content of tests/campaign/content.yml through romfetch, and runs core_probe for the files, boot, path, alias, variants and bench checks. A fixed rule table settles hle_fallback, required, path and aliases with the record ids as evidence, the rest goes to measurements/findings.jsonl. Resumable state, load, disk and budget guards, retries, content reference counting with an LRU cap, and an autorun job after the nightly.

scripts/autorun.py

@@ -54,6 +54,7 @@ JOBS_FILE = REPO_ROOT / "automation" / "jobs.yml"
 AUTORUN_DIR = Path(".cache") / "autorun"
 DEFAULT_LOCK = Path.home() / ".local" / "state" / "retrobios" / "auto.lock"
 LOCK_ENV = "RETROBIOS_AUTO_LOCK"
+LOCK_HELD_ENV = "RETROBIOS_AUTO_LOCK_HELD"
 
 BRANCH = "autorun"
 REMOTE = "origin"
@@ -726,6 +727,9 @@ def set_alert(state: dict, name: str, active: bool, detail: str, now: str) -> No
 def job_env(job: Job) -> dict[str, str]:
     env = os.environ.copy()
     env["RETROBIOS_AUTO"] = "1"
+    if job.lock:
+        # A step that takes the host lock itself (campaign.py) runs under ours.
+        env[LOCK_HELD_ENV] = "1"
     if job.network:
         sys.path.insert(0, str(REPO_ROOT / "scripts"))
         from common import github_token

scripts/campaign_profiles.py

@@ -0,0 +1,161 @@
+"""Write a settled field into one entry of a profile, and nothing else.
+
+The profile is edited as text, so its comments, order and quoting stay as
+the maintainer wrote them. Every edit is checked by parsing the result: the
+edited entry must equal the old one with the new fields, and every other
+part of the profile must parse to what it did before. An edit that cannot
+hold both is refused (ProfileEditError), never written.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import yaml
+
+ITEM = re.compile(r"^(?P<indent>\s*)- ")
+KEY = re.compile(r"^(?P<indent>\s*)(?P<key>[A-Za-z_][A-Za-z0-9_]*):(?P<rest>.*)$")
+# Where a new key goes: after the first of these the entry has, else last.
+ANCHORS = {
+    "required": ("name",),
+    "hle_fallback": ("required", "name"),
+    "path": ("name",),
+    "aliases": ("path", "name"),
+}
+
+
+class ProfileEditError(ValueError):
+    """An edit the profile's layout does not allow to make safely."""
+
+
+def scalar(value: object) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if value is None:
+        return "null"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return json.dumps(value, ensure_ascii=False)
+    if isinstance(value, list):
+        return "[" + ", ".join(scalar(v) for v in value) + "]"
+    raise ProfileEditError(f"no scalar form for {value!r}")
+
+
+def item_spans(lines: list[str]) -> list[tuple[int, int, int]]:
+    """(first line, end line, item indent) of each item of the top-level files: list."""
+    try:
+        start = next(i for i, line in enumerate(lines) if line.rstrip() == "files:")
+    except StopIteration as exc:
+        raise ProfileEditError("no top-level files: list") from exc
+    spans: list[tuple[int, int, int]] = []
+    indent: int | None = None
+    current: int | None = None
+    end = len(lines)
+    for i in range(start + 1, len(lines)):
+        line = lines[i]
+        stripped = line.strip()
+        if not stripped or stripped.startswith("#"):
+            continue
+        width = len(line) - len(line.lstrip(" "))
+        if width == 0 and not line.startswith("-"):
+            end = i
+            break
+        match = ITEM.match(line)
+        if match and (indent is None or width == indent):
+            indent = width
+            if current is not None:
+                spans.append((current, i, indent))
+            current = i
+    if current is not None:
+        spans.append((current, end, indent or 0))
+    # An item ends at its last line of content: the comments and blank lines
+    # before the next item belong to that item.
+    trimmed = []
+    for first, stop, width in spans:
+        last = stop
+        while last > first + 1 and (not lines[last - 1].strip() or lines[last - 1].strip().startswith("#")):
+            last -= 1
+        trimmed.append((first, last, width))
+    return trimmed
+
+
+def key_lines(lines: list[str], first: int, stop: int, key_indent: int) -> dict[str, tuple[int, int]]:
+    """key -> (line, end line) for each key of one item, its continuation lines included."""
+    keys: dict[str, tuple[int, int]] = {}
+    order: list[tuple[str, int]] = []
+    for i in range(first, stop):
+        # The item's first line carries the dash where its first key's indent is.
+        line = " " * key_indent + lines[i][key_indent:] if i == first else lines[i]
+        match = KEY.match(line)
+        if match and len(match["indent"]) == key_indent:
+            order.append((match["key"], i))
+    for position, (key, line) in enumerate(order):
+        end = order[position + 1][1] if position + 1 < len(order) else stop
+        while end > line + 1 and (not lines[end - 1].strip() or lines[end - 1].strip().startswith("#")):
+            end -= 1
+        keys[key] = (line, end)
+    return keys
+
+
+def _render(key: str, value: object, indent: int, block: bool = False) -> list[str]:
+    """One key as lines. A list keeps the block style it was written in."""
+    pad = " " * indent
+    if isinstance(value, dict):
+        return [f"{pad}{key}:"] + [f"{pad}  {k}: {scalar(v)}" for k, v in value.items()]
+    if block and isinstance(value, list):
+        return [f"{pad}{key}:"] + [f"{pad}  - {scalar(v)}" for v in value]
+    return [f"{pad}{key}: {scalar(value)}"]
+
+
+def set_fields(text: str, index: int, updates: dict[str, object]) -> str:
+    """The profile with these keys of files[index] set, the rest untouched."""
+    before = yaml.safe_load(text)
+    entries = before.get("files") or []
+    if not 0 <= index < len(entries):
+        raise ProfileEditError(f"no files[{index}]")
+    lines = text.splitlines()
+    spans = item_spans(lines)
+    if len(spans) != len(entries):
+        raise ProfileEditError(f"{len(spans)} items read as text, {len(entries)} parsed")
+    first, stop, item_indent = spans[index]
+    key_indent = item_indent + 2
+    for key, value in updates.items():
+        keys = key_lines(lines, first, stop, key_indent)
+        if key in keys:
+            start, end = keys[key]
+            rendered = _render(key, value, key_indent, block=lines[start].rstrip().endswith(":"))
+            if start == first:
+                raise ProfileEditError(f"files[{index}]: {key} is the item's first key")
+            lines[start:end] = rendered
+            stop += len(rendered) - (end - start)
+            continue
+        rendered = _render(key, value, key_indent)
+        anchor = next((keys[a][1] for a in ANCHORS.get(key, ()) if a in keys), None)
+        at = anchor if anchor is not None else stop
+        lines[at:at] = rendered
+        stop += len(rendered)
+    out = "\n".join(lines) + ("\n" if text.endswith("\n") else "")
+    after = yaml.safe_load(out)
+    expected = {**entries[index], **updates}
+    if after.get("files", [None] * (index + 1))[index] != expected:
+        raise ProfileEditError(f"files[{index}]: the edit does not parse to the intended entry")
+    rest_before = {k: v for k, v in before.items() if k != "files"}
+    rest_after = {k: v for k, v in after.items() if k != "files"}
+    others_before = [e for i, e in enumerate(entries) if i != index]
+    others_after = [e for i, e in enumerate(after.get("files") or []) if i != index]
+    if rest_before != rest_after or others_before != others_after:
+        raise ProfileEditError(f"files[{index}]: the edit changes another part of the profile")
+    return out
+
+
+def write_fields(path: Path, index: int, updates: dict[str, object]) -> bool:
+    """Apply the edit to the file. False when it changes nothing."""
+    text = path.read_text(encoding="utf-8")
+    out = set_fields(text, index, updates)
+    if out == text:
+        return False
+    path.write_text(out, encoding="utf-8")
+    return True

scripts/campaign_rules.py

@@ -0,0 +1,203 @@
+"""The campaign's fixed rule table: which observable settles which profile field.
+
+Each rule is a pure function of a core probe record (or what was read from
+one) and a profile entry. It returns verdicts. A verdict with `apply` set is
+settled by the table and written into the profile with the record ids as
+`evidence:`. Any other verdict goes to the review queue
+(measurements/findings.jsonl).
+
+  observable                                        field          settled
+  boots to the game with no file                    hle_fallback   true
+  no game screen without it, the game with it       required       true
+                                                    hle_fallback   false
+  opens the staged file under another path          path           that path
+  opens the staged file under another name          aliases        name added
+  a staged file the content should use, never read  -              flagged only
+
+"Boots to the game" is read from the sampled frames: the run without the
+file shows at least MIN_SHARED_SCREENS distinct screens that the run with
+the file also shows, single-colour frames excluded. "No game screen" is a
+run that did not load, or whose every sampled frame is a single colour. A
+run that shows screens of its own that the reference never shows settles
+nothing.
+
+Standard library only.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, field
+
+MIN_SHARED_SCREENS = 2
+BOOTS = "boots"
+FAILS = "fails"
+INCONCLUSIVE = "inconclusive"
+
+# A field an entry leaves out reads as this value.
+FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": []}
+SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases")
+
+
+@dataclass(frozen=True)
+class Boot:
+    """What the runs without the file showed, against the runs with it."""
+
+    status: str
+    detail: str
+    shared: int = 0
+
+
+@dataclass(frozen=True)
+class Verdict:
+    rule: str
+    field: str
+    value: object
+    statement: str
+    evidence: tuple[str, ...]
+    apply: bool = False
+
+
+@dataclass(frozen=True)
+class Opened:
+    """Where a run found the staged file opened."""
+
+    dest: str
+    record_id: str
+    extra: dict = field(default_factory=dict)
+
+
+def screens(repetition: dict) -> set[str]:
+    """The distinct sampled frames of a repetition that are not one colour."""
+    observables = repetition.get("observables") or {}
+    uniform = set(observables.get("uniform_frames") or ())
+    return {digest for _, digest in observables.get("frame_hashes") or () if digest not in uniform}
+
+
+def variant(record: dict, label: str) -> dict:
+    for item in record.get("variants") or ():
+        if item.get("label") == label:
+            return item
+    raise KeyError(f"record {record.get('id')}: no variant {label!r}")
+
+
+def boot_outcome(record: dict, absent_label: str, present_label: str) -> Boot:
+    """Whether the core reached the game without the file.
+
+    Every repetition must agree: one that boots and one that does not leave
+    the question open.
+    """
+    present = variant(record, present_label)["repetitions"]
+    if any(not rep.get("loaded") for rep in present):
+        return Boot(INCONCLUSIVE, "with the file, the core did not load the content")
+    reference: set[str] = set()
+    for rep in present:
+        shown = screens(rep)
+        if not shown:
+            return Boot(INCONCLUSIVE, "with the file, the core showed no game screen")
+        reference |= shown
+    outcomes: list[str] = []
+    shared_counts: list[int] = []
+    for rep in variant(record, absent_label)["repetitions"]:
+        if not rep.get("loaded"):
+            outcomes.append(FAILS)
+            continue
+        shown = screens(rep)
+        if not shown:
+            outcomes.append(FAILS)
+            continue
+        shared = len(shown & reference)
+        shared_counts.append(shared)
+        outcomes.append(BOOTS if shared >= MIN_SHARED_SCREENS else INCONCLUSIVE)
+    shared = min(shared_counts) if shared_counts else 0
+    if all(o == BOOTS for o in outcomes):
+        return Boot(BOOTS, f"without the file, every run showed {shared} or more screens the "
+                           f"run with the file shows", shared)
+    if all(o == FAILS for o in outcomes):
+        return Boot(FAILS, "without the file, no run loaded the content or showed more than "
+                           "one colour, while the run with the file reached the game")
+    return Boot(INCONCLUSIVE, "without the file, the runs showed screens the run with the file "
+                              "never shows, or disagreed with each other", shared)
+
+
+def current(entry: dict, name: str) -> object:
+    if name in entry:
+        return entry[name]
+    return FIELD_DEFAULTS.get(name)
+
+
+def agrees(entry: dict, verdict: Verdict) -> bool:
+    """True when the profile already says what the run showed."""
+    if verdict.field == "aliases":
+        return set(verdict.value or ()) <= set(entry.get("aliases") or ())
+    return current(entry, verdict.field) == verdict.value
+
+
+def rule_bios_need(boot: Boot, entry: dict, record_id: str) -> list[Verdict]:
+    """Boot with no file against boot with the declared file."""
+    name = entry["name"]
+    evidence = (record_id,)
+    if boot.status == BOOTS:
+        out = [Verdict("boots_without_file", "hle_fallback", True,
+                       f"the core reaches the game without {name}: {boot.detail}", evidence, True)]
+        if entry.get("required") is True:
+            out.append(Verdict(
+                "boots_without_file", "required", False,
+                f"the profile says {name} is required, and the core reaches the game "
+                f"without it. The table settles hle_fallback from this run, not required",
+                evidence))
+        return out
+    if boot.status == FAILS:
+        return [
+            Verdict("fails_without_file", "required", True,
+                    f"the core reaches the game only with {name}: {boot.detail}", evidence, True),
+            Verdict("fails_without_file", "hle_fallback", False,
+                    f"no stand-in for {name}: {boot.detail}", evidence, True),
+        ]
+    return [Verdict("inconclusive", "hle_fallback", None,
+                    f"the runs with and without {name} settle nothing: {boot.detail}", evidence)]
+
+
+def rule_path(entry: dict, declared: str, opened: Opened) -> Verdict | None:
+    """The core opened the file at another place under the system directory."""
+    if opened.dest == declared:
+        return None
+    return Verdict("opened_under_other_path", "path", opened.dest,
+                   f"the core opened {entry['name']} as {opened.dest}, the profile declares "
+                   f"{declared}", (opened.record_id,), True)
+
+
+def rule_alias(entry: dict, opened: Opened) -> Verdict | None:
+    """The core opened the file under a name the entry does not declare."""
+    name = opened.dest.rsplit("/", 1)[-1]
+    known = [entry["name"], *(entry.get("aliases") or ())]
+    if name in known:
+        return None
+    return Verdict("opened_other_name", "aliases", [*(entry.get("aliases") or ()), name],
+                   f"the core opened {entry['name']} under the name {name}",
+                   (opened.record_id,), True)
+
+
+def rule_never_opened(entry: dict, record_ids: Sequence[str], content: str) -> Verdict:
+    return Verdict("never_opened", "opened", False,
+                   f"{entry['name']} was staged and the core never opened it while running "
+                   f"{content}", tuple(record_ids))
+
+
+def rule_variants(status: str, labels: Iterable[str], entry: dict, record_id: str) -> Verdict | None:
+    """Copies of one file that change what the core shows: a pack choice to review."""
+    if status != "differs":
+        return None
+    return Verdict("variant_changes_output", "variants", sorted(labels),
+                   f"the copies of {entry['name']} change what the core shows",
+                   (record_id,))
+
+
+def merge_evidence(existing: dict | None, verdict: Verdict) -> dict:
+    """The entry's evidence: map with the verdict's record ids added under its field."""
+    out = {key: list(value) for key, value in (existing or {}).items()}
+    ids = out.setdefault(verdict.field, [])
+    for rid in verdict.evidence:
+        if rid not in ids:
+            ids.append(rid)
+    return {key: out[key] for key in sorted(out)}

scripts/core_probe.py

@@ -269,6 +269,14 @@ def render_printf(fmt: bytes, args: Sequence[int]) -> str:
     return "".join(out).rstrip("\n")
 
 
+def uniform_frame(frame: bytes, pixel_bytes: int) -> bool:
+    """True for a frame of one colour, an empty one included: a screen that
+    shows nothing of the game."""
+    if len(frame) <= pixel_bytes:
+        return True
+    return frame == frame[:pixel_bytes] * (len(frame) // pixel_bytes)
+
+
 def png_bytes(width: int, height: int, rgb: bytes) -> bytes:
     def chunk(tag: bytes, body: bytes) -> bytes:
         return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body))
@@ -359,6 +367,7 @@ class Frontend:
         self.video = hashlib.sha1()
         self.audio = hashlib.sha1()
         self.frame_hashes: list[list] = []
+        self.uniform: set[str] = set()
         self.logs: list[dict] = []
         self.presses: list[tuple[int, int, int]] = [tuple(p) for p in spec["presses"]]
         self.unanswered: dict[int, int] = {}
@@ -433,7 +442,10 @@ class Frontend:
         self.video.update(struct.pack("<II", *self.last_size) + hashlib.sha1(self.last_frame).digest())
         every = self.spec["frame_hash_every"]
         if every and self.frame % every == 0:
-            self.frame_hashes.append([self.frame, hashlib.sha1(self.last_frame).hexdigest()])
+            digest = hashlib.sha1(self.last_frame).hexdigest()
+            self.frame_hashes.append([self.frame, digest])
+            if digest not in self.uniform and uniform_frame(self.last_frame, PIXEL_BYTES[self.pixel_format]):
+                self.uniform.add(digest)
             png_dir = self.spec.get("png_dir")
             if png_dir and self.last_frame:
                 rgb = to_rgb(self.pixel_format, *self.last_size, self.last_frame)
@@ -514,6 +526,7 @@ def run_worker(spec: dict) -> dict:
         observables[f"state:{name}"] = state_vars.get(name)
     if spec["frame_hash_every"]:
         observables["frame_hashes"] = front.frame_hashes
+        observables["uniform_frames"] = sorted(front.uniform)
     core.retro_unload_game()
     core.retro_deinit()
     return {
@@ -584,7 +597,7 @@ def frontend_identity() -> dict:
 
 def core_identity(args: argparse.Namespace) -> dict:
     core = Path(args.core)
-    return {
+    identity = {
         "file": core.name,
         "sha256": file_digest(core, "sha256"),
         "repo": args.core_repo,
@@ -592,6 +605,9 @@ def core_identity(args: argparse.Namespace) -> dict:
         "build_command": args.build_command,
         "compiler": elf_comment(core) or None,
     }
+    if args.core_artifact:
+        identity["artifact"] = args.core_artifact
+    return identity
 
 
 # --------------------------------------------------------------------------
@@ -769,6 +785,8 @@ def build_parser() -> argparse.ArgumentParser:
     common.add_argument("--core-repo", help="source repository URL of the core build")
     common.add_argument("--core-commit", help="commit the core was built from")
     common.add_argument("--build-command", help="command and flags that built the core")
+    common.add_argument("--core-artifact", metavar="URL",
+                        help="where the .so was downloaded from, such as a buildbot nightly")
     common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
                         help="append the record as one line of measurements/<emulator>.jsonl "
                              "in the checkout DIR (default: this one)")
@@ -841,7 +859,7 @@ def validate_args(args: argparse.Namespace) -> list[str]:
                 problems.append(f"--vary {dest}={src}: not a file")
         names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var}
         if args.frame_hash_every:
-            names.add("frame_hashes")
+            names.update(("frame_hashes", "uniform_frames"))
         if args.observable not in names:
             problems.append(f"--observable {args.observable}: not one of {', '.join(sorted(names))}")
     return problems
@@ -936,8 +954,9 @@ def default_subject(record: dict) -> str:
     return f"{record['kind']}-{slugify(content, 30)}-{digest}"
 
 
-def write_record(directory: Path, full: dict) -> Path:
-    """Append the record, with its id, to its emulator's .jsonl."""
+def write_record(directory: Path, full: dict) -> tuple[Path, dict]:
+    """Append the record, with its id, to its emulator's .jsonl. Returns the
+    file and the record as written."""
     from measurement_records import line_text, parse_lines, records_path, seal  # noqa: PLC0415
 
     sealed = seal(full)
@@ -948,7 +967,7 @@ def write_record(directory: Path, full: dict) -> Path:
     lines.parent.mkdir(parents=True, exist_ok=True)
     with lines.open("a", encoding="utf-8") as handle:
         handle.write(line_text(sealed) + "\n")
-    return lines
+    return lines, sealed
 
 
 def probe(args: argparse.Namespace) -> int:
@@ -999,7 +1018,7 @@ def probe(args: argparse.Namespace) -> int:
             "repetitions": repetitions,
             "presses": [{"frame": f, "button": b, "length": n} for f, b, n in args.press],
             "observables": list(BASE_OBSERVABLES) + [f"state:{v}" for v in args.state_var]
-                           + (["frame_hashes"] if args.frame_hash_every else []),
+                           + (["frame_hashes", "uniform_frames"] if args.frame_hash_every else []),
             "frame_hash_every": args.frame_hash_every,
             "joypad_ports": args.joypad_ports,
         },
@@ -1025,7 +1044,7 @@ def probe(args: argparse.Namespace) -> int:
     record["subject"] = args.subject or default_subject(record)
     if args.record:
         try:
-            lines = write_record(Path(args.record), record)
+            lines, record = write_record(Path(args.record), record)
         except ProbeError as exc:
             print(f"ERROR: {exc}", file=sys.stderr)
             return 2

scripts/measurement_records.py

@@ -81,6 +81,47 @@ def records_path(emulator: str) -> str:
     return f"{RECORDS_DIR}/{emulator}.jsonl"
 
 
+# The campaign's own files beside the records: append-only like them, one
+# line per finding or per pass, each line hashed to its id. No profile takes
+# these names (tests/test_campaign.py checks it).
+FINDINGS = "findings"
+CAMPAIGN_RUNS = "campaign-runs"
+CAMPAIGN_FILES = {
+    FINDINGS: ("found_at", "campaign-finding.schema.json"),
+    CAMPAIGN_RUNS: ("started_at", "campaign-run.schema.json"),
+}
+
+
+def line_id(line: dict, date_key: str) -> str:
+    body = {key: value for key, value in line.items() if key != "id"}
+    return f"{line[date_key][:10]}-{sha256_hex(canonical(body))[:12]}"
+
+
+def seal_line(line: dict, date_key: str) -> dict:
+    return {**line, "id": line_id(line, date_key)}
+
+
+def read_campaign(root: Path, name: str) -> list[dict]:
+    """The lines of one campaign file, each checked against its id."""
+    path = root / records_path(name)
+    if not path.is_file():
+        return []
+    date_key = CAMPAIGN_FILES[name][0]
+    lines = parse_lines(path.read_text(encoding="utf-8"), records_path(name))
+    problems = []
+    seen: set[str] = set()
+    for number, line in enumerate(lines, 1):
+        rid = str(line.get("id", ""))
+        if not ID_PATTERN.match(rid) or date_key not in line or line_id(line, date_key) != rid:
+            problems.append(f"{records_path(name)}:{number}: id {rid!r} does not hash the line")
+        if rid in seen:
+            problems.append(f"{records_path(name)}:{number}: id {rid} appears twice")
+        seen.add(rid)
+    if problems:
+        raise MeasurementsError("\n  ".join(problems))
+    return lines
+
+
 IDENTITY = ("id", "emulator", "subject", "supersedes", "migrated_from_version")
 
 
@@ -177,6 +218,8 @@ def read_records(root: Path) -> list[Stored]:
         if not path.endswith(".jsonl") or "/" in emulator:
             problems.append(f"{path}: {RECORDS_DIR}/ holds <emulator>.jsonl files only")
             continue
+        if emulator in CAMPAIGN_FILES:
+            continue
         seen: set[str] = set()
         try:
             records = parse_lines(files[path].read_text(encoding="utf-8"), path)

scripts/sitemeasurements.py

@@ -140,6 +140,80 @@ def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
     }
 
 
+def campaign_summary(root: Path, stored: list[mr.Stored], source: dict | None) -> dict:
+    """The last campaign pass and the findings still waiting for review.
+
+    A finding's history is its lines under one key: the latest line is its
+    state, and only an open or flagged one is listed.
+    """
+    runs = mr.read_campaign(root, mr.CAMPAIGN_RUNS)
+    latest: dict[str, dict] = {}
+    applied = 0
+    for line in mr.read_campaign(root, mr.FINDINGS):
+        latest[line["key"]] = line
+        applied += line["status"] == "applied"
+    where = {s.record["id"]: s for s in stored}
+    links = {}
+    for line in latest.values():
+        for rid in line["evidence"]:
+            found = where.get(rid)
+            if found and source and source.get("commit"):
+                links[rid] = permalink(source["repo"], source["commit"],
+                                       mr.records_path(found.emulator), found.line)
+    open_items = [{**line, "links": [links.get(rid) for rid in line["evidence"]]}
+                  for _, line in sorted(latest.items()) if line["status"] in ("open", "flagged")]
+    return {"last_run": runs[-1] if runs else None, "passes": len(runs), "applied": applied,
+            "open": open_items}
+
+
+def _value(value: object) -> str:
+    return f"`{json.dumps(value)}`" if value is not None else "nothing"
+
+
+def campaign_section(summary: dict | None) -> list[str]:
+    """The campaign's last pass and its open findings, for the explorer page."""
+    if not summary or not summary.get("last_run"):
+        return []
+    last = summary["last_run"]
+    counts = last["counts"]
+    lines = [
+        "## Campaign",
+        "",
+        f"Tests run by themselves every night on an idle computer ([how]({METHOD_PAGE})). "
+        f"Last pass: {last['ended_at'][:10]}, {_text(last['result'])}"
+        + (f" ({_text(last['detail'])})" if last.get("detail") else "") + ". "
+        f"{counts['done']} tests done, {counts['failed']} failed, {counts['pending']} waiting. "
+        f"{summary['applied']} profile fields were set from a run.",
+        "",
+    ]
+    if last["pending_reasons"]:
+        lines += ["| Waiting for | Tests |", "|-------------|------:|"]
+        lines += [f"| {_text(reason)} | {number} |"
+                  for reason, number in sorted(last["pending_reasons"].items(), key=lambda kv: (-kv[1], kv[0]))]
+        lines.append("")
+    if last["failed"]:
+        lines += ["Failed tests:", ""]
+        lines += [f"- {_text(item['emulator'])}, `{_text(item['check'])}`: {_text(item['reason'])}"
+                  for item in last["failed"]]
+        lines.append("")
+    lines += ["### Open findings", ""]
+    if not summary["open"]:
+        lines += ["None: every finding was settled by a rule or resolved by a later run.", ""]
+        return lines
+    lines += ["A finding is what a run showed that no rule settles. It waits for someone to "
+              "read the emulator's code.", ""]
+    for item in summary["open"]:
+        records = ", ".join(f"[{rid}]({link})" if link else f"`{rid}`"
+                            for rid, link in zip(item["evidence"], item["links"]))
+        lines += [
+            f"- **{_text(item['emulator'])}**, `{_text(item['file'])}`, `{_text(item['field'])}` "
+            f"({item['status']}): the profile says {_value(item['profile_says'])}, the run shows "
+            f"{_value(item['run_shows'])}. {_text(item['statement'])}. Records: {records}.",
+            "",
+        ]
+    return lines
+
+
 # --------------------------------------------------------------------------
 # SQLite
 # --------------------------------------------------------------------------
@@ -360,6 +434,7 @@ def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulato
     source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)}
     profiles = load_profiles(emulators_dir)
     index = build_index(stored, source, profiles, system_names)
+    index["campaign"] = campaign_summary(root, stored, source)
     paths = [public / INDEX_PATH]
     (public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
     write_if_changed(str(public / INDEX_PATH),
@@ -485,6 +560,7 @@ def generate_explorer_page(index: dict) -> str:
         "",
         "<MeasurementsExplorer />",
         "",
+        *campaign_section(index.get("campaign")),
     ])
 
 

scripts/validate_schemas.py

@@ -601,6 +601,8 @@ def _measurement_errors(root: Path | None = None) -> list[str]:
     root = root or ROOT
     validator = _validator("core-probe-record.schema.json")
     out: list[str] = []
+    campaign = {mr.records_path(name): _validator(schema)
+                for name, (_, schema) in mr.CAMPAIGN_FILES.items()}
     for path, file in sorted(mr.tree_files(root).items()):
         if not path.endswith(".jsonl"):
             continue
@@ -610,10 +612,12 @@ def _measurement_errors(root: Path | None = None) -> list[str]:
             except json.JSONDecodeError as exc:
                 out.append(f"{path}:{number}: {exc}")
                 continue
-            out.extend(_errors(validator, record, f"{path}:{number}"))
+            out.extend(_errors(campaign.get(path, validator), record, f"{path}:{number}"))
     if not out:
         try:
             mr.read_records(root)
+            for name in mr.CAMPAIGN_FILES:
+                mr.read_campaign(root, name)
         except mr.MeasurementsError as exc:
             out.append(str(exc))
     return out