chore: merge the boot oracle work

scripts/campaign.py

@@ -1805,6 +1805,8 @@ def main(argv: Sequence[str] | None = None) -> int:
     scope.add_argument("--romfetch", default="",
                        help=f"romfetch command (default: ${ROMFETCH_ENV}, else romfetch on PATH)")
     scope.add_argument("--offline", action="store_true", help="no buildbot, no romfetch, no build")
+    scope.add_argument("--content-file", default=str(CONTENT_FILE),
+                       help="the test content list (default: tests/campaign/content.yml)")
     scope.add_argument("--allow-unsandboxed", action="store_true",
                        help="run cores and builds without bubblewrap when it is missing (refused by default)")
     run_p = sub.add_parser("run", parents=[scope], help="one pass: cores, content, checks, rules")
@@ -1862,6 +1864,8 @@ def main(argv: Sequence[str] | None = None) -> int:
     elif args.command == "references":
         settings = Settings(romfetch=args.romfetch, network=not args.offline,
                             allow_unsandboxed=args.allow_unsandboxed)
+    if args.command in ("run", "references"):
+        paths.content_file = Path(args.content_file)
     campaign = Campaign(paths, settings)
     if args.command == "status":
         report = status_report(campaign)

scripts/core_probe.py

@@ -1197,8 +1197,8 @@ def judge(rep: dict, oracle: dict) -> tuple[str, dict]:
     """The verdict of one repetition and the evidence that decided it.
 
     In order: a worker killed after the timeout is a hang and one that died
-    a crash, from the process status. A reference frame reached is a boot.
-    Without one, an error pattern of the core log or a known error screen
+    a crash, from the process status. A reference frame of the game reached
+    is a boot: the console's own screens alone are not. Without one, an error pattern of the core log or a known error screen
     is a BIOS error. Every sampled frame of the second half one colour is a
     hang. Anything else is unknown: a run is never called a boot because
     its frames differ from another run's.
@@ -1217,12 +1217,23 @@ def judge(rep: dict, oracle: dict) -> tuple[str, dict]:
         frames.append((int(rep.get("frames", 0)) + int(rep.get("warmup", 0)), final))
     references = {r["sha1"]: r for r in oracle.get("references") or ()}
     reached = [(frame, digest) for frame, digest in frames if digest in references]
-    if reached:
-        frame, digest = reached[0]
-        stages = list(dict.fromkeys(references[d]["stage"] for _, d in reached))
+    stages = list(dict.fromkeys(references[d]["stage"] for _, d in reached))
+    game = [(frame, digest) for frame, digest in reached if references[digest]["stage"] == "game"]
+    if game:
+        frame, digest = game[0]
         return BOOTS, {"rule": "reference_frame",
                        "detail": f"frame {frame} shows {references[digest]['shows']}",
                        "frame": frame, "sha1": digest, "stages": stages}
+    verdict, evidence = _without_the_game(rep, oracle, frames)
+    if stages:
+        # The console's own screens without one of the game: never a boot.
+        evidence["stages"] = stages
+        evidence["detail"] += ", after the BIOS screens and no screen of the game"
+    return verdict, evidence
+
+
+def _without_the_game(rep: dict, oracle: dict, frames: list[tuple[int, str]]) -> tuple[str, dict]:
+    """The verdict of a run that reached no screen of the game."""
     for pattern in oracle.get("error_patterns") or ():
         for line in rep.get("logs") or ():
             if re.search(pattern, line["text"]):

scripts/sitemeasurements.py

@@ -39,7 +39,7 @@ HOSTS_MARKER = "<!-- measurements-hosts -->"
 INDEX_PATH = Path("data") / "measurements" / "index.json"
 SQLITE_PATH = Path("downloads") / "measurements.sqlite"
 PER_EMULATOR_DIR = Path("downloads") / "measurements"
-SQLITE_SCHEMA_VERSION = "2"
+SQLITE_SCHEMA_VERSION = "3"
 # Observables the variants table leaves out: one value per sampled frame.
 BULKY_OBSERVABLES = ("frame_hashes",)
 INDEX_SCHEMA_VERSION = 1
@@ -74,6 +74,40 @@ def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str
     return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
 
 
+def verdicts_of(record: dict) -> list[dict]:
+    """Each variant's boot verdict, from record version 4 on."""
+    return [{"variant": v["label"], "verdict": v["verdict"],
+             "detail": (v.get("verdict_evidence") or {}).get("detail", "")}
+            for v in record["variants"] if "verdict" in v]
+
+
+def result_of(record: dict) -> tuple[str, str]:
+    """The status and statement a page shows for a record.
+
+    A run, or a check compared on the sampled frames, is shown by its boot
+    verdicts: one verdict when every variant has it, else `mixed`, each
+    variant named in the statement. A check on another observable keeps
+    its comparison. A frame comparison recorded before the boot oracle
+    shows `no verdict`: frames that differ prove nothing about a boot.
+    """
+    check = record.get("check") or {}
+    result = check.get("result") or {}
+    if record["kind"] == "bench":
+        return "measured", bench_statement(record) if record.get("timing") else ""
+    verdicts = verdicts_of(record)
+    by_frames = record["kind"] == "run" or check.get("observable") == "frame_hashes"
+    if verdicts and by_frames:
+        values = {v["verdict"] for v in verdicts}
+        status = values.pop() if len(values) == 1 else "mixed"
+        if record["kind"] == "run":
+            return status, verdicts[0]["detail"]
+        return status, ". ".join(f"{v['variant']}: {v['verdict']}" for v in verdicts)
+    if by_frames:
+        return "no verdict", ("recorded before the boot oracle. Its frames were compared, which "
+                              "settles nothing about a boot")
+    return result.get("status") or "ran", result.get("statement") or ""
+
+
 def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
                system_names: dict[str, str]) -> dict:
     record = entry.record
@@ -93,9 +127,9 @@ def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
         "method": check.get("method"),
         "observable": check.get("observable"),
         "variable": check.get("variable"),
-        "status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
-        "statement": (check.get("result") or {}).get("statement")
-                     or (bench_statement(record) if record.get("timing") else ""),
+        "status": result_of(record)[0],
+        "statement": result_of(record)[1],
+        "verdicts": verdicts_of(record),
         "content": content.get("label"),
         "variants": len(record["variants"]),
         "repetitions": record["inputs"]["repetitions"],
@@ -434,7 +468,8 @@ SCHEMA_SQL = """
         run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
         label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
         flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
-        files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
+        files_probed_absent TEXT, tracing TEXT, verdict TEXT, verdict_evidence TEXT,
+        PRIMARY KEY (run_id, position)
     );
     CREATE TABLE system_files (
         run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
@@ -545,11 +580,12 @@ def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
         # The first repetition stands for the variant, its per-frame hashes left out.
         observables = {key: value for key, value in (first.get("observables") or {}).items()
                        if key not in BULKY_OBSERVABLES}
-        connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
+        connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
             rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)),
             json_text(variant["flags"]), json_text(observables),
             json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")),
-            json_text(first.get("tracing"))))
+            json_text(first.get("tracing")), variant.get("verdict"),
+            json_text(variant.get("verdict_evidence")) if "verdict_evidence" in variant else None))
         connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
             (rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
     for metric, stats in sorted((r.get("timing") or {}).items()):
@@ -646,12 +682,14 @@ def tested_item(item: dict) -> list[str]:
     core = item["core"]
     build = f"{_text(core['library'])} at {_short(core['commit'])}" if core.get("commit") \
         else _text(core["library"])
-    lines = [
-        f"- **{_text(title)}**",
-        "",
-        f"    **Result:** `{item['status']}`. {_text(item['statement'])}.",
-        "",
-    ]
+    lines = [f"- **{_text(title)}**", ""]
+    if len(item.get("verdicts") or ()) > 1:
+        lines += [f"    **Result:** `{item['status']}`.", ""]
+        lines += [f"    - {_text(v['variant'])}: `{v['verdict']}`. {_text(v['detail'])}."
+                  for v in item["verdicts"]]
+        lines.append("")
+    else:
+        lines += [f"    **Result:** `{item['status']}`. {_text(item['statement'])}.", ""]
     if item.get("method"):
         lines += [f"    **Method:** {_text(item['method'])}", ""]
     lines += [

tests/campaign/oracle.yml

@@ -21,6 +21,151 @@
 
 error_patterns: []
 
-cores: {}
+cores:
+  # The PlayStation cores below log a missing or wrong BIOS and boot a
+  # built-in one: none of those lines is an error pattern.
+  "Beetle PSX":
+    note: >-
+      A missing BIOS logs "Firmware is missing" and loads the embedded
+      OpenBIOS, a wrong one logs "Firmware found but has invalid SHA1" and is
+      loaded anyway (libretro.c:849, 856, 3504-3511 at 7fea57a5).
+    error_patterns: []
+    error_screens: []
+  "Beetle PSX HW":
+    note: The same code as Beetle PSX, built with the hardware renderers.
+    error_patterns: []
+    error_screens: []
+  "SwanStation":
+    note: >-
+      "No BIOS image found for <region> region" is followed by the built-in
+      OpenBIOS (src/core/host_interface.cpp:182, src/core/system.cpp:678-682
+      at 631f0fb3).
+    error_patterns: []
+    error_screens: []
+  "PCSX-ReARMed":
+    note: >-
+      Without a BIOS, or with one of the wrong size, it logs "Using HLE BIOS"
+      and runs its own (frontend/libretro.c:3893-3899 at c8816799). The BIOS
+      boot logo is off by default (pcsx_rearmed_show_bios_bootlogo), so its
+      runs reach no bios-stage screen.
+    error_patterns: []
+    error_screens: []
+  "PCSX1":
+    note: An HLE BIOS stands in when none is found, and the run logs nothing about it.
+    error_patterns: []
+    error_screens: []
 
-content: []
+content:
+  - name: "Bust-A-Move 2 - Arcade Edition (USA)"
+    catalog: redump
+    # The sha1 of the .cue handed to the core.
+    sha1: ed4a437a6f653cc71ad6ac0da68cc85eb093f298
+    references:
+      "Beetle PSX":
+        recorded_from: 2026-10-27-b0150a15670d
+        frames:
+          - sha1: 7b72ebae83893b63f9fffd1685f03b1d048d9fa9
+            stage: bios
+            shows: the Sony Computer Entertainment logo
+          - sha1: 0905c920a1de996664eddca94f93f75d0dcd4c52
+            stage: bios
+            shows: the PlayStation logo licensed by Sony Computer Entertainment America
+          - sha1: d51ab7783f8eeafc63a28f14ce39f8e2babeffb3
+            stage: game
+            shows: Now Loading with the yellow bubble
+          - sha1: 63a96bd6d1d487da1824e10d37f6658506edc451
+            stage: game
+            shows: the Acclaim and Taito credits text, CREDIT 6
+          - sha1: 6628416f377d2e25df45e5fa2126f8f9313551f0
+            stage: game
+            shows: the Acclaim logo, CREDIT 6
+          - sha1: d68892362e0f33c41cc59dac1b5a2077fe5428ee
+            stage: game
+            shows: the Taito logo on white, CREDIT 6
+          - sha1: 56330c6c7646c2c261acb3f48450825cb289845a
+            stage: game
+            shows: a black screen with CREDIT 6
+      "Beetle PSX HW":
+        recorded_from: 2026-10-27-ee69ad83c9f8
+        frames:
+          - sha1: 7b72ebae83893b63f9fffd1685f03b1d048d9fa9
+            stage: bios
+            shows: the Sony Computer Entertainment logo
+          - sha1: 0905c920a1de996664eddca94f93f75d0dcd4c52
+            stage: bios
+            shows: the PlayStation logo licensed by Sony Computer Entertainment America
+          - sha1: d51ab7783f8eeafc63a28f14ce39f8e2babeffb3
+            stage: game
+            shows: Now Loading with the yellow bubble
+          - sha1: 63a96bd6d1d487da1824e10d37f6658506edc451
+            stage: game
+            shows: the Acclaim and Taito credits text, CREDIT 6
+          - sha1: 6628416f377d2e25df45e5fa2126f8f9313551f0
+            stage: game
+            shows: the Acclaim logo, CREDIT 6
+          - sha1: d68892362e0f33c41cc59dac1b5a2077fe5428ee
+            stage: game
+            shows: the Taito logo on white, CREDIT 6
+          - sha1: 56330c6c7646c2c261acb3f48450825cb289845a
+            stage: game
+            shows: a black screen with CREDIT 6
+      "SwanStation":
+        recorded_from: 2026-10-27-717f780131fc
+        frames:
+          - sha1: fad859d9b8211ce9f35ad3a05b0b7d184fe14d83
+            stage: bios
+            shows: the Sony Computer Entertainment logo
+          - sha1: 593a6e2600946040fefd6eb9aebdc5c0d56f0555
+            stage: bios
+            shows: the PlayStation logo licensed by Sony Computer Entertainment America
+          - sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
+            stage: game
+            shows: Now Loading with the yellow bubble
+          - sha1: dd6dac23da6b8f4f256b8090568c52e8f8428425
+            stage: game
+            shows: the Acclaim and Taito credits text, CREDIT 6
+          - sha1: 533715dd946ba1984af1b9356e06989c7770e04a
+            stage: game
+            shows: the Acclaim logo, CREDIT 6
+          - sha1: 0d17db6e9a2d5793695884c85dd2177093420fdc
+            stage: game
+            shows: the Taito logo on white, CREDIT 6
+          - sha1: 2238516839dc43c0b22c00db5fe76128f5fb6f3f
+            stage: game
+            shows: a black screen with CREDIT 6
+      "PCSX-ReARMed":
+        recorded_from: 2026-10-27-8154bab7bf80
+        frames:
+          - sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
+            stage: game
+            shows: Now Loading with the yellow bubble
+          - sha1: dd6dac23da6b8f4f256b8090568c52e8f8428425
+            stage: game
+            shows: the Acclaim and Taito credits text, CREDIT 6
+          - sha1: 533715dd946ba1984af1b9356e06989c7770e04a
+            stage: game
+            shows: the Acclaim logo, CREDIT 6
+          - sha1: 0d17db6e9a2d5793695884c85dd2177093420fdc
+            stage: game
+            shows: the Taito logo on white, CREDIT 6
+          - sha1: 2238516839dc43c0b22c00db5fe76128f5fb6f3f
+            stage: game
+            shows: a black screen with CREDIT 6
+      "PCSX1":
+        recorded_from: 2026-10-27-2905a9dd4b0a
+        frames:
+          - sha1: e5af1f2023d0d729ce7aabf52d8e03f75bb8d230
+            stage: game
+            shows: Now Loading with the yellow bubble
+          - sha1: dd6dac23da6b8f4f256b8090568c52e8f8428425
+            stage: game
+            shows: the Acclaim and Taito credits text, CREDIT 6
+          - sha1: e10f642259f2906bb39ae07fcb4b5a36eae18287
+            stage: game
+            shows: the Acclaim logo with a stray strip at its right edge, CREDIT 6
+          - sha1: 0d17db6e9a2d5793695884c85dd2177093420fdc
+            stage: game
+            shows: the Taito logo on white, CREDIT 6
+          - sha1: 2238516839dc43c0b22c00db5fe76128f5fb6f3f
+            stage: game
+            shows: a black screen with CREDIT 6

tests/test_campaign.py

@@ -882,6 +882,19 @@ class Findings(Workspace):
         self.assertIn("would commit 'chore: add campaign measurements'", out.getvalue())
 
 
+class ContentFile(Workspace):
+    def test_a_content_list_can_be_named(self) -> None:
+        other = self.tmp / "other.yml"
+        other.write_text(yaml.safe_dump({"systems": {"fake-system": {
+            "romfetch": "fakesys", "content": [{"name": "Other Game (Japan)", "catalog": "redump",
+                                                "reason": "r"}]}}}), encoding="utf-8")
+        out = io.StringIO()
+        with unittest.mock.patch.object(campaign.Paths, "default", return_value=self.paths), \
+                contextlib.redirect_stdout(out):
+            self.assertEqual(campaign.main(["run", "--dry-run", "--content-file", str(other)]), 0)
+        self.assertIn("Other Game (Japan)", out.getvalue())
+
+
 class InstallTimer(unittest.TestCase):
     def test_writes_the_units_and_enables_only_when_asked(self) -> None:
         calls = []

tests/test_core_probe.py

@@ -209,7 +209,15 @@ class OracleTests(unittest.TestCase):
                      logs=["BIOS not found"]), self.ORACLE)
         self.assertEqual(verdict, "boots")
         self.assertEqual((evidence["rule"], evidence["frame"], evidence["stages"]),
-                         ("reference_frame", 10, ["bios", "game"]))
+                         ("reference_frame", 20, ["bios", "game"]))
+
+    def test_the_bios_screens_alone_are_not_a_boot(self) -> None:
+        # A console that shows its logo and then refuses the disc never reached the game.
+        verdict, evidence = core_probe.judge(
+            self.rep(frames((0, self.LOGO), (10, self.BLACK), (20, self.BLACK)), uniform=[self.BLACK]),
+            self.ORACLE)
+        self.assertEqual((verdict, evidence["rule"], evidence["stages"]), ("hang", "uniform_tail", ["bios"]))
+        self.assertIn("no screen of the game", evidence["detail"])
 
     def test_an_error_pattern_or_screen_without_a_reference_is_a_bios_error(self) -> None:
         verdict, evidence = core_probe.judge(

tests/test_measurements.py

@@ -220,7 +220,7 @@ class ReadAndIndex(Workspace):
         keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind",
                 "claim", "method", "observable", "variable", "status", "statement", "content",
                 "variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id",
-                "runs", "record_url", "sqlite"}
+                "runs", "record_url", "sqlite", "verdicts"}
         for item in index["items"]:
             self.assertEqual(set(item), keys)
         alpha = {i["subject"]: i for i in index["items"] if i["emulator"] == "alpha"}
@@ -234,6 +234,25 @@ class ReadAndIndex(Workspace):
         written = json.loads((self.tmp / "public" / sm.INDEX_PATH).read_text())
         self.assertEqual(written, index)
 
+    def test_a_frame_check_shows_its_verdicts_never_frames_that_differ(self) -> None:
+        record = full_record("alpha", "boot", "2026-10-06T10:00:00+00:00", observable="frame_hashes")
+        record["variants"][1]["verdict"] = "hang"
+        self.assertEqual(sm.result_of(record), ("mixed", "v0: boots. v1: hang"))
+        self.assertEqual([v["verdict"] for v in sm.verdicts_of(record)], ["boots", "hang"])
+        old = {**record, "record_version": 3,
+               "variants": [{k: v for k, v in variant.items() if not k.startswith("verdict")}
+                            for variant in record["variants"]]}
+        status, statement = sm.result_of(old)
+        self.assertEqual(status, "no verdict")
+        self.assertNotIn("differ", statement)
+        # A check on another observable keeps its comparison.
+        self.assertEqual(sm.result_of(full_record("alpha", "c", "2026-10-06T10:00:00+00:00"))[0], "differs")
+        item = {"claim": "c", "runs": 1, "core": {"library": "Fake 1", "commit": None}, "status": "mixed",
+                "statement": "", "verdicts": sm.verdicts_of(record), "method": "m", "host": "h",
+                "date": "2026-10-06", "variants": 2, "repetitions": 2, "flags": 0, "record_url": "u"}
+        text = "\n".join(sm.tested_item(item))
+        self.assertIn("v1: `hang`", text)
+
     def test_links_name_the_last_commit_that_touched_the_records(self) -> None:
         first = self.repository()
         (self.tree / "other.txt").write_text("x")

wiki/measurements.md

@@ -14,25 +14,34 @@ the emulator's page shows a **Tested** block. Every result is on the
 ## What a test does
 
 - A script runs the emulator with no window and no sound.
-- It starts a game or a disc and presses buttons at set frames.
-- It runs again for each version of one file, such as a BIOS. Nothing else changes.
-- It compares what the emulator showed, played or saved, by fingerprint (a hash).
-- Each version runs at least twice, each time from a fresh start.
-
-## When tests run
-
-A maintainer starts a test run. It then works alone on an idle computer. A clear
-result updates the emulator's page and names the test as proof.
-
-Anything else waits on the [Measurements](../measurements.md) page for someone
-to read the code.
+- It starts a game and runs it for a set time.
+- Each run starts fresh and runs at least twice.
+- Each run gets one result, by comparing its screens with ones we checked by eye.
 
 ## What a result says
 
-- **same**: the file changed nothing that was compared.
-- **differs**: some versions of the file changed it.
-- **inconclusive**: two runs of one version did not match, so no answer is given.
+- **boots**: the game reached a screen it shows when it works.
+- **bios_error**: the emulator said a BIOS is missing or wrong.
+- **crash**: the emulator stopped.
+- **hang**: the screen stayed blank, or the run never ended.
+- **unknown**: none of these could be shown.
 - **measured**: a speed test, given as time per frame.
+- **no verdict**: an older test that only compared screens.
+
+## How a file is proven
+
+The game runs with no BIOS, with the BIOS alone, and with a wrong file under
+its name: zeros, half the file, or another region's BIOS.
+
+- **Required**: no BIOS fails, the BIOS alone boots.
+- **Built-in stand-in**: no BIOS still boots.
+- **Checked**: a wrong file is refused.
+
+## When tests run
+
+A maintainer starts a test run. A clear result updates the emulator's page and
+names the test as proof. Anything else waits on the
+[Measurements](../measurements.md) page for someone to read the code.
 
 ## What a result keeps
 

wiki/tools.md

@@ -706,10 +706,12 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6
 
 ### autorun.py
 
-Runs the unattended jobs declared in `automation/jobs.yml`, one at a time.
-The registry holds each job's commands, its gate, the artifacts computed
-twice, the paths each commit may carry and the schedule. Changing any of
-them is a reviewed commit.
+Runs the jobs declared in `automation/jobs.yml`, one at a time. Nothing is
+scheduled by default: a maintainer runs a job by hand, and a contributor who
+wants a schedule on their own machine installs the timers below. The registry
+holds each job's commands, its gate, the artifacts computed twice, the paths
+each commit may carry and the schedule. Changing any of them is a reviewed
+commit.
 
 | Guard | Rule |
 |-------|------|
@@ -845,6 +847,12 @@ python scripts/core_probe.py bench --core core.so --content game.cue \
 python scripts/core_probe.py check --core core.so --content disc.cue \
   --vary mpeg_card.bin=a.bin --vary mpeg_card.bin=b.bin --vary mpeg_card.bin= \
   --observable video_sha1 --claim "..." --method "..." --record
+
+# negative variants: the file, zeros of its size, its first half
+python scripts/core_probe.py check --core core.so --content disc.cue \
+  --frame-hash-every 10 --vary scph5501.bin=bios/scph5501.bin \
+  --vary scph5501.bin=@zeros:524288 --vary scph5501.bin=@head:262144:bios/scph5501.bin \
+  --observable frame_hashes --claim "..." --method "..." --repeat 2 --record
 ```
 
 Each repetition runs in a fresh process under `strace -f -e trace=openat,open`
@@ -859,15 +867,41 @@ writable. Every namespace is unshared, the network included, so nothing of
 the user (`~/.config`, `~/.ssh`, `~/.git-credentials`) or of the checkout is
 there to read. Each repetition names what it ran under in `isolation`
 (`bwrap`, or `environment only` with the reason), and `--require-sandbox`
-refuses to run without it. A timeout kills the worker's whole process group. The files the core opened under the system
+refuses to run without it. The files the core opened under the system
 directory come from the kernel, each hashed again after the run. Without
 strace the record says that tracing was unavailable.
 
+**Boot oracle.** Every repetition and every variant carries a `verdict` and
+the `verdict_evidence` that decided it (record version 4), judged against
+`tests/campaign/oracle.yml` (`--oracle`), in this order:
+
+| Verdict | Evidence |
+|---------|----------|
+| `hang` | the worker was killed after `--timeout` (`timeout`) |
+| `crash` | the worker died, by exit status or signal (`exit_status`). Its logs and sampled frames are rebuilt from the events it streamed |
+| `boots` | a sampled frame or the last frame is a `game` reference frame of this content on this core (`reference_frame`), with the `stages` reached: `bios` for a screen only the console's BIOS draws, `game` for one of the content. The BIOS screens alone are never a boot: the stages go into the evidence of the verdict below |
+| `bios_error` | a log line matches an error pattern of the core (`error_pattern`), or a frame is one of its error screens (`error_screen`) |
+| `hang` | every sampled frame of the second half is one colour (`uniform_tail`) |
+| `unknown` | none of these (`no_evidence`) |
+
+A variant's verdict is the one all its repetitions agree on, else `unknown`
+(`repetitions_disagree`). A check lists them in `check.result.verdicts`, and
+`inputs.oracle` keeps what they were judged against. The oracle finds a
+content by the sha1 of the file handed to the core and a core by its library
+name. A reference frame is recorded once with the full BIOS set and reviewed
+by looking at its PNG. An error pattern is a log line after which the core
+stops, read from its source: a line a core logs before it falls back to a
+built-in BIOS is not one. Frames that differ from another run never make a
+boot.
+
 | Option | Effect |
 |--------|--------|
 | `--system-file DEST=SRC` | copy SRC into a staged system directory as DEST |
 | `--content-label` | identity of the content, such as its Redump name, copied into the record |
-| `--work-dir`, `--timeout` | scratch directory for the staged runs, and seconds allowed per repetition (default 1800) |
+| `--work-dir`, `--timeout` | scratch directory for the staged runs, and seconds allowed per repetition (default 1800), after which the run is killed and judged a hang |
+| `--oracle FILE` | reference frames, error patterns and error screens the verdicts are judged against (default `tests/campaign/oracle.yml`) |
+| `--repeat N` | runs of each variant, each in a fresh process (`run` defaults to 1, `check` to 2, `bench` to 5) |
+| `--vary DEST=SRC` | (`check`) one variant per occurrence. An empty SRC leaves DEST absent, `@zeros:SIZE` stages SIZE zero bytes and `@head:SIZE:PATH` the first SIZE bytes of PATH |
 | `--option KEY=VALUE` | core option. Unset options take the first value the core declares |
 | `--frames`, `--warmup` | frames timed, and frames run before timing starts |
 | `--press FRAME:BUTTON[:LEN]` | hold a libretro joypad button on port 0 (`a`, `b`, `start`, `up`...). A menu may need 15 frames or more to see it |
@@ -947,6 +981,9 @@ files a core opens nor writes a record.
 python scripts/campaign.py run --system sony-playstation --budget 6h
 python scripts/campaign.py run --only beetle_psx,swanstation --dry-run
 python scripts/campaign.py run --commit --commit-every 10
+python scripts/campaign.py run --only pcsx_rearmed --bench   # with benchmarks
+python scripts/campaign.py references --only beetle_psx --system sony-playstation
+python scripts/campaign.py references --only beetle_saturn --content-file other.yml   # another content list
 python scripts/campaign.py status            # done, failed, pending, and why
 python scripts/campaign.py findings          # what waits for review
 python scripts/campaign.py gate              # the checks behind a commit
@@ -957,18 +994,37 @@ python scripts/campaign.py install-timer     # opt-in: writes the autorun units,
 |------|--------------|
 | Core | The x86_64 Linux nightly from `buildbot.libretro.com`, for each name of the profile's `cores:` the buildbot publishes, downloaded again only when its `ETag` or `Last-Modified` changes. The record keeps the artifact URL, the `.so` sha256 and the `retro_get_system_info` version. With no nightly, the profile's `source_commit` is cloned and built once with `make` (`Makefile.libretro` first), the failure kept so it is not tried again for that commit. Host git only fetches into bare mirrors (`.cache/campaign/mirrors/`), one per repository, submodules included, that it alone touches: no working tree, never bound into a sandbox, and earlier working-tree clones are deleted. It runs with no user or system config, https as the only transport, and `-c` overrides emptying hooks, `core.fsmonitor`, `core.sshCommand`, `core.gitProxy`, credential helpers and every filter driver the mirror's config defines. The commit and each submodule at the commit its tree records are exported with `git archive` into a fresh folder, and `make` runs there in the same bubblewrap sandbox as a core, that folder the one writable path. The core copied out is the newest `*_libretro.so` that is a regular file inside that folder, opened without following links. The record's `build_command` says which isolation was used. A standalone emulator is `pending: standalone` |
 | Content | Each system's games come from `tests/campaign/content.yml`, by catalog name and with the reason each was chosen. They are read from `.cache/test-content/<romfetch system>/<name>/`, each file checked against its `identity.json`. Missing content is fetched with `romfetch get <system> <name> --json` when romfetch is on `PATH`, in `RETROBIOS_ROMFETCH` or given with `--romfetch`. Otherwise the checks are `pending: content`, with romfetch's exit code and error code when it failed. Content never enters git, and a record keeps only its identity |
-| Checks | `files` stages every declared file the collection holds and reads which ones the core opens (strace). `boot` runs the game with no file and with the declared one (`boot_file:`, else the entry of the game's region). An entry the `files` run left unread is staged `alone`, then where the core looked for it (`path`) and under the names it looked for (`alias`). `variants` runs each `.variants/` copy of a file the core opened. `bench` times the game, warm-up excluded, 5 repetitions |
+| Checks | `files` stages every declared file the collection holds (the full set) and reads which ones the core opens (strace). `none` runs the game with no BIOS at all. `negative` runs, for each declared file of the game's region the collection holds, the file alone, then zeros of its size, its first half and up to two other regions' dumps staged under its name. An entry neither the full set nor its own run opened is staged where the core looked for it (`path`), else flagged. The `boot_file:` entry is also staged under the names the core looked for with no BIOS (`alias`). `variants` runs each `.variants/` copy of a file the core opened. Each runs twice from a fresh process and is judged by the boot oracle. `bench` (only with `--bench`) times the game once the full set boots, warm-up excluded, 5 repetitions |
 | Rules | `campaign_rules.py`, a fixed table of pure functions (below). A field the table settles is written into the profile, with `evidence:` naming the record ids. Anything else is a line of `measurements/findings.jsonl` |
 | Content reuse | A game stays on disk until the last emulator that needs it in the pass is done, then the campaign deletes it if it fetched it. Past `--cache-cap-gb` (default 20) the least recently used goes first. An entry marked `pinned:`, or one the campaign did not fetch, is never deleted |
 
-| Observable | Field | Written |
-|------------|-------|---------|
-| Without the file, at least 2 screens the run with the file also shows (single-colour frames left out), in every repetition | `hle_fallback` | `true` |
-| Without the file, no repetition loads the game or shows more than one colour, while the run with it reaches the game | `required`, `hle_fallback` | `true`, `false` |
+| Verdicts | Field | Written |
+|----------|-------|---------|
+| `none` boots | `hle_fallback` | `true` |
+| `none` fails (`bios_error`, `crash`, `hang`) and the file alone boots | `required`, `hle_fallback` | `true`, `false` |
+| Zeros of the file's size refused while the file boots | `validation` | a content check is proven. Agreeing adds evidence, a profile with no md5, sha1, sha256, crc32 or adler32 is an open finding |
+| Zeros used while the file boots | `validation` | no content check. A profile that declares one is an open finding |
+| Its first half refused while zeros are used | `validation` | `size`, added when missing |
+| Its first half used | `validation` | no size check. A profile that declares one is an open finding |
 | The core opened the file staged at another place it looked for | `path` | that path |
 | The core opened the file staged under a name no entry declares | `aliases` | the name added |
 | A staged file the game's region uses, never opened, staged alone either | nothing | flagged |
-| Anything else (runs that disagree, a profile that says `required: true` for a file the core boots without) | nothing | open finding |
+| Copies of one file whose verdicts differ | nothing | open finding |
+| Anything else (an `unknown`, a profile that says `required: true` for a file the core boots without) | nothing | open finding |
+
+A wrong file is refused when the run reports `bios_error`, or boots without
+the `bios` screens the right file shows: the core fell back to a built-in
+BIOS. It is used when it turns the run into a `hang` or a `crash` that the
+run with no BIOS does not show. Another region's dump is recorded with its
+verdict, and the region claims are settled by their own test. Evidence that a
+record from before the oracle wrote no longer counts: when a verdict settles a
+field again, those ids leave its `evidence:`.
+
+`references` runs the full set once per core and content with every sampled
+frame written as PNG under `tmp/campaign/references/<core>/<content>/`, and
+prints the record id with the screens that stay on for 3 samples or more in
+every repetition. A reviewer opens those PNGs and writes the chosen frames
+into `tests/campaign/oracle.yml` with their stage and what they show.
 
 A run that agrees with the profile adds its record id to `evidence:` and
 resolves an open finding of the same field. The profile is edited as text and
@@ -977,8 +1033,9 @@ those fields changed.
 
 State lives in `.cache/campaign/state.json` of the main checkout: each check
 is `done`, `failed` (with the reason and its log) or `pending` (with the
-reason), with the core sha256 and the profile digest it ran against. A pass
-skips a check that is done or failed against the same core and profile, so it
+reason), with the core sha256, the profile digest, the oracle sha256 and the
+record version it ran against. A pass skips a check that is done or failed
+against the same four, so it
 takes new nightlies, changed profiles, newly fetched content and pending
 checks. The campaign's own `evidence:` writes do not count as a profile change.
 
@@ -990,8 +1047,8 @@ checks. The campaign's own `evidence:` writes do not count as a profile change.
 | Load | Before each run, waits while the 1-minute load is above a quarter of the CPUs, up to `--max-load-wait` seconds, then the pass stops as `paused` |
 | Disk | Below `--min-free-gb` (default 30) the pass stops as `paused` |
 | Budget | No check starts after `--budget` |
-| Timeout | A run is killed after 900 seconds per repetition and variant, recorded `failed: timeout`, and the pass moves on |
-| Crash | A run that dies is tried 3 times, then `failed` with its log |
+| Timeout | A repetition is killed after 900 seconds and judged `hang`. A core_probe call that outlives all its repetitions is `failed: timeout` |
+| Crash | A core that dies is a `crash` verdict in its record. A core_probe call that fails for another reason is tried 3 times, then `failed` with its log |
 | Network | Buildbot and romfetch source errors are retried after 10, 60 and 300 seconds |
 
 Each pass appends one line to `measurements/campaign-runs.jsonl`: when it ran,