chore: merge the boot oracle work

scripts/campaign.py

@@ -1805,6 +1805,8 @@ def main(argv: Sequence[str] | None = None) -> int:
     scope.add_argument("--romfetch", default="",
                        help=f"romfetch command (default: ${ROMFETCH_ENV}, else romfetch on PATH)")
     scope.add_argument("--offline", action="store_true", help="no buildbot, no romfetch, no build")
+    scope.add_argument("--content-file", default=str(CONTENT_FILE),
+                       help="the test content list (default: tests/campaign/content.yml)")
     scope.add_argument("--allow-unsandboxed", action="store_true",
                        help="run cores and builds without bubblewrap when it is missing (refused by default)")
     run_p = sub.add_parser("run", parents=[scope], help="one pass: cores, content, checks, rules")
@@ -1862,6 +1864,8 @@ def main(argv: Sequence[str] | None = None) -> int:
     elif args.command == "references":
         settings = Settings(romfetch=args.romfetch, network=not args.offline,
                             allow_unsandboxed=args.allow_unsandboxed)
+    if args.command in ("run", "references"):
+        paths.content_file = Path(args.content_file)
     campaign = Campaign(paths, settings)
     if args.command == "status":
         report = status_report(campaign)

scripts/core_probe.py

@@ -1197,8 +1197,8 @@ def judge(rep: dict, oracle: dict) -> tuple[str, dict]:
     """The verdict of one repetition and the evidence that decided it.
 
     In order: a worker killed after the timeout is a hang and one that died
-    a crash, from the process status. A reference frame reached is a boot.
-    Without one, an error pattern of the core log or a known error screen
+    a crash, from the process status. A reference frame of the game reached
+    is a boot: the console's own screens alone are not. Without one, an error pattern of the core log or a known error screen
     is a BIOS error. Every sampled frame of the second half one colour is a
     hang. Anything else is unknown: a run is never called a boot because
     its frames differ from another run's.
@@ -1217,12 +1217,23 @@ def judge(rep: dict, oracle: dict) -> tuple[str, dict]:
         frames.append((int(rep.get("frames", 0)) + int(rep.get("warmup", 0)), final))
     references = {r["sha1"]: r for r in oracle.get("references") or ()}
     reached = [(frame, digest) for frame, digest in frames if digest in references]
-    if reached:
-        frame, digest = reached[0]
-        stages = list(dict.fromkeys(references[d]["stage"] for _, d in reached))
+    stages = list(dict.fromkeys(references[d]["stage"] for _, d in reached))
+    game = [(frame, digest) for frame, digest in reached if references[digest]["stage"] == "game"]
+    if game:
+        frame, digest = game[0]
         return BOOTS, {"rule": "reference_frame",
                        "detail": f"frame {frame} shows {references[digest]['shows']}",
                        "frame": frame, "sha1": digest, "stages": stages}
+    verdict, evidence = _without_the_game(rep, oracle, frames)
+    if stages:
+        # The console's own screens without one of the game: never a boot.
+        evidence["stages"] = stages
+        evidence["detail"] += ", after the BIOS screens and no screen of the game"
+    return verdict, evidence
+
+
+def _without_the_game(rep: dict, oracle: dict, frames: list[tuple[int, str]]) -> tuple[str, dict]:
+    """The verdict of a run that reached no screen of the game."""
     for pattern in oracle.get("error_patterns") or ():
         for line in rep.get("logs") or ():
             if re.search(pattern, line["text"]):

scripts/sitemeasurements.py

@@ -39,7 +39,7 @@ HOSTS_MARKER = "<!-- measurements-hosts -->"
 INDEX_PATH = Path("data") / "measurements" / "index.json"
 SQLITE_PATH = Path("downloads") / "measurements.sqlite"
 PER_EMULATOR_DIR = Path("downloads") / "measurements"
-SQLITE_SCHEMA_VERSION = "2"
+SQLITE_SCHEMA_VERSION = "3"
 # Observables the variants table leaves out: one value per sampled frame.
 BULKY_OBSERVABLES = ("frame_hashes",)
 INDEX_SCHEMA_VERSION = 1
@@ -74,6 +74,40 @@ def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str
     return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
 
 
+def verdicts_of(record: dict) -> list[dict]:
+    """Each variant's boot verdict, from record version 4 on."""
+    return [{"variant": v["label"], "verdict": v["verdict"],
+             "detail": (v.get("verdict_evidence") or {}).get("detail", "")}
+            for v in record["variants"] if "verdict" in v]
+
+
+def result_of(record: dict) -> tuple[str, str]:
+    """The status and statement a page shows for a record.
+
+    A run, or a check compared on the sampled frames, is shown by its boot
+    verdicts: one verdict when every variant has it, else `mixed`, each
+    variant named in the statement. A check on another observable keeps
+    its comparison. A frame comparison recorded before the boot oracle
+    shows `no verdict`: frames that differ prove nothing about a boot.
+    """
+    check = record.get("check") or {}
+    result = check.get("result") or {}
+    if record["kind"] == "bench":
+        return "measured", bench_statement(record) if record.get("timing") else ""
+    verdicts = verdicts_of(record)
+    by_frames = record["kind"] == "run" or check.get("observable") == "frame_hashes"
+    if verdicts and by_frames:
+        values = {v["verdict"] for v in verdicts}
+        status = values.pop() if len(values) == 1 else "mixed"
+        if record["kind"] == "run":
+            return status, verdicts[0]["detail"]
+        return status, ". ".join(f"{v['variant']}: {v['verdict']}" for v in verdicts)
+    if by_frames:
+        return "no verdict", ("recorded before the boot oracle. Its frames were compared, which "
+                              "settles nothing about a boot")
+    return result.get("status") or "ran", result.get("statement") or ""
+
+
 def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
                system_names: dict[str, str]) -> dict:
     record = entry.record
@@ -93,9 +127,9 @@ def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
         "method": check.get("method"),
         "observable": check.get("observable"),
         "variable": check.get("variable"),
-        "status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
-        "statement": (check.get("result") or {}).get("statement")
-                     or (bench_statement(record) if record.get("timing") else ""),
+        "status": result_of(record)[0],
+        "statement": result_of(record)[1],
+        "verdicts": verdicts_of(record),
         "content": content.get("label"),
         "variants": len(record["variants"]),
         "repetitions": record["inputs"]["repetitions"],
@@ -434,7 +468,8 @@ SCHEMA_SQL = """
         run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
         label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
         flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
-        files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
+        files_probed_absent TEXT, tracing TEXT, verdict TEXT, verdict_evidence TEXT,
+        PRIMARY KEY (run_id, position)
     );
     CREATE TABLE system_files (
         run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
@@ -545,11 +580,12 @@ def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
         # The first repetition stands for the variant, its per-frame hashes left out.
         observables = {key: value for key, value in (first.get("observables") or {}).items()
                        if key not in BULKY_OBSERVABLES}
-        connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
+        connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
             rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)),
             json_text(variant["flags"]), json_text(observables),
             json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")),
-            json_text(first.get("tracing"))))
+            json_text(first.get("tracing")), variant.get("verdict"),
+            json_text(variant.get("verdict_evidence")) if "verdict_evidence" in variant else None))
         connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
             (rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
     for metric, stats in sorted((r.get("timing") or {}).items()):
@@ -646,12 +682,14 @@ def tested_item(item: dict) -> list[str]:
     core = item["core"]
     build = f"{_text(core['library'])} at {_short(core['commit'])}" if core.get("commit") \
         else _text(core["library"])
-    lines = [
-        f"- **{_text(title)}**",
-        "",
-        f"    **Result:** `{item['status']}`. {_text(item['statement'])}.",
-        "",
-    ]
+    lines = [f"- **{_text(title)}**", ""]
+    if len(item.get("verdicts") or ()) > 1:
+        lines += [f"    **Result:** `{item['status']}`.", ""]
+        lines += [f"    - {_text(v['variant'])}: `{v['verdict']}`. {_text(v['detail'])}."
+                  for v in item["verdicts"]]
+        lines.append("")
+    else:
+        lines += [f"    **Result:** `{item['status']}`. {_text(item['statement'])}.", ""]
     if item.get("method"):
         lines += [f"    **Method:** {_text(item['method'])}", ""]
     lines += [