chore: merge the boot oracle

tests/campaign/oracle.yml

@@ -0,0 +1,26 @@
+# The boot oracle of scripts/core_probe.py: what a run must reach to boot,
+# and what tells a BIOS error apart.
+#
+# content       per piece of test content (by the sha1 of the file handed to
+#               the core, the .cue for a disc), per core (by the library name
+#               retro_get_system_info gives): the reference frames a run with
+#               the full BIOS set reaches. Each was recorded once
+#               (`recorded_from`, the record id) and reviewed by looking at its
+#               PNG. `stage: bios` is a screen only the console's own BIOS
+#               draws, `stage: game` a screen of the content. A run that reaches
+#               one of them boots.
+# error_patterns  log lines after which a core stops, read from its source
+#               (`source_ref`). Shared ones at the top level, a core's own under
+#               cores:. A line a core logs and then carries on (a fallback to a
+#               built-in BIOS) is not an error pattern.
+# error_screens per core, the sha1 of a frame that shows its BIOS error message.
+#
+# A run that reaches no reference frame, matches no pattern or screen and
+# does not end on one colour is `unknown`: frames that differ from another
+# run's never make a boot.
+
+error_patterns: []
+
+cores: {}
+
+content: []

tests/test_campaign.py

@@ -16,6 +16,7 @@ import io
 import json
 import os
 import shutil
+import struct
 import subprocess
 import sys
 import tempfile
@@ -92,15 +93,20 @@ void retro_init(void) {}
 void retro_deinit(void) {}
 static FILE *try_open(const char *dir, const char *name) {
   char path[4096]; snprintf(path, sizeof path, "%s/%s", dir, name); return fopen(path, "rb"); }
+/* booted: 0 a black screen to the end, 1 the BIOS logo then the title,
+   2 the title alone (the built-in stand-in of FAKE_HLE). */
 bool retro_load_game(const void *g) {
-  const char *dir = NULL; FILE *f; (void)g; env(9, &dir);
+  const char *dir = NULL; FILE *f; unsigned char buf[8]; size_t n = 0; (void)g; env(9, &dir);
   f = try_open(dir, "bios.bin");
   if (!f) f = try_open(dir, "BIOS.BIN");
+  if (f) { n = fread(buf, 1, sizeof buf, f); fclose(f); }
+  int valid = f && n == 4 && buf[0] == 'B';
 #ifdef FAKE_HLE
-  if (f) fclose(f);
-  booted = 1;
+  /* Checks the size only: a file of the wrong size is refused for the
+     stand-in, one of the right size is run, whatever it holds. */
+  booted = valid ? 1 : (f && n == 4) ? 0 : 2;
 #else
-  if (f) { fclose(f); booted = 1; } else booted = 0;
+  booted = valid ? 1 : 0;
 #endif
   return true; }
 void retro_unload_game(void) {}
@@ -109,7 +115,8 @@ void retro_run(void) {
 #ifdef FAKE_HANG
   for (;;) {}
 #endif
-  for (int i = 0; i < 8; i++) pixels[i] = booted ? (uint16_t)(frame / 2 + i) : 0;
+  int shown = frame < 4 ? 0 : booted == 1 && frame < 8 ? 1 : booted ? 2 : 0;
+  for (int i = 0; i < 8; i++) pixels[i] = shown == 0 ? 0 : (uint16_t)((shown == 1 ? 1 : 20) + i);
   frame++;
   video(pixels, 4, 2, 8); batch(s, 2); }
 size_t retro_serialize_size(void) { return 0; }
@@ -117,78 +124,94 @@ bool retro_serialize(void *d, size_t n) { (void)d; (void)n; return false; }
 """
 
 
-def rep(frames: list[str], uniform: list[str] | None = None, loaded: bool = True) -> dict:
-    return {"loaded": loaded, "observables": {
-        "frame_hashes": [[i * 10, h] for i, h in enumerate(frames)],
-        "uniform_frames": sorted(set(uniform or []))}}
+def run(verdict: str, label: str = "v", stages: tuple[str, ...] = (),
+        rid: str = "2026-10-27-aaaaaaaaaaaa") -> rules.Run:
+    return rules.Run(verdict, rid, label, stages)
 
 
-def boot_record(absent: list[dict], present: list[dict], rid: str = "2026-10-27-aaaaaaaaaaaa") -> dict:
-    return {"id": rid, "variants": [{"label": "bios.bin-absent", "repetitions": absent},
-                                    {"label": "bios.bin", "repetitions": present}]}
-
-
-class BootOutcome(unittest.TestCase):
-    GAME = ("black", "logo", "title", "title2")
-
-    def test_boots_when_screens_are_shared(self) -> None:
-        record = boot_record([rep(["black", "hle", "logo", "title"], ["black"])] * 2,
-                             [rep(self.GAME, ["black"])] * 2)
-        boot = rules.boot_outcome(record, "bios.bin-absent", "bios.bin")
-        self.assertEqual((boot.status, boot.shared), (rules.BOOTS, 2))
-
-    def test_one_colour_and_no_load_fail(self) -> None:
-        black = boot_record([rep(["black"], ["black"]), rep([], [], loaded=False)],
-                            [rep(self.GAME, ["black"])])
-        self.assertEqual(rules.boot_outcome(black, "bios.bin-absent", "bios.bin").status, rules.FAILS)
-
-    def test_a_shared_blank_screen_proves_nothing(self) -> None:
-        # The black frame both runs start on is one colour: it is no game screen.
-        record = boot_record([rep(["black", "error"], ["black"])], [rep(self.GAME, ["black"])])
-        self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
-                         rules.INCONCLUSIVE)
-
-    def test_one_shared_screen_is_not_enough(self) -> None:
-        record = boot_record([rep(["logo", "error"])], [rep(self.GAME, ["black"])])
-        self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
-                         rules.INCONCLUSIVE)
-
-    def test_repetitions_that_disagree_settle_nothing(self) -> None:
-        record = boot_record([rep(["logo", "title"]), rep(["black"], ["black"])],
-                             [rep(self.GAME, ["black"])])
-        self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
-                         rules.INCONCLUSIVE)
-
-    def test_a_reference_that_shows_nothing_settles_nothing(self) -> None:
-        record = boot_record([rep(["logo", "title"])], [rep(["black"], ["black"])])
-        self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
-                         rules.INCONCLUSIVE)
-        record = boot_record([rep(["logo", "title"])], [rep(self.GAME, loaded=False)])
-        self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
-                         rules.INCONCLUSIVE)
+NONE_BOOTS = run(rules.BOOTS, "none", ("game",), "2026-10-27-000000000001")
+NONE_HANGS = run(rules.HANG, "none", (), "2026-10-27-000000000001")
+PRESENT = run(rules.BOOTS, "bios.bin", ("bios", "game"), "2026-10-27-000000000002")
 
 
 class RuleTable(unittest.TestCase):
     ENTRY = MappingProxyType({"name": "bios.bin", "required": True, "aliases": ["Bios.bin"]})
     RID = "2026-10-27-bbbbbbbbbbbb"
 
-    def test_boots_without_settles_hle_and_queues_required(self) -> None:
-        verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "x", 3), self.ENTRY, self.RID)
+    def test_a_verdict_is_read_from_the_record(self) -> None:
+        record = {"id": self.RID, "variants": [
+            {"label": "a", "verdict": "boots", "verdict_evidence": {"rule": "repetitions_agree",
+                                                                    "detail": "d", "stages": ["bios"]}},
+            {"label": "old"}]}
+        self.assertEqual(rules.run_of(record, "a"), rules.Run("boots", self.RID, "a", ("bios",)))
+        # A record from before the oracle proves nothing.
+        self.assertEqual(rules.run_of(record, "old").verdict, rules.UNKNOWN)
+
+    def test_boots_with_no_bios_settles_hle_and_queues_required(self) -> None:
+        verdicts = rules.rule_need(NONE_BOOTS, PRESENT, self.ENTRY)
         self.assertEqual([(v.field, v.value, v.apply) for v in verdicts],
                          [("hle_fallback", True, True), ("required", False, False)])
-        self.assertEqual(verdicts[0].evidence, (self.RID,))
-        optional = rules.rule_bios_need(rules.Boot(rules.BOOTS, "x", 3),
-                                        {"name": "a", "required": False}, self.RID)
+        self.assertEqual(verdicts[0].evidence, (NONE_BOOTS.record_id,))
+        optional = rules.rule_need(NONE_BOOTS, PRESENT, {"name": "a", "required": False})
         self.assertEqual([v.field for v in optional], ["hle_fallback"])
 
-    def test_fails_without_settles_required_and_no_hle(self) -> None:
-        verdicts = rules.rule_bios_need(rules.Boot(rules.FAILS, "x"), self.ENTRY, self.RID)
-        self.assertEqual([(v.field, v.value, v.apply) for v in verdicts],
-                         [("required", True, True), ("hle_fallback", False, True)])
+    def test_fails_without_and_boots_alone_settles_required(self) -> None:
+        for failure in rules.FAILS:
+            verdicts = rules.rule_need(run(failure, "none"), PRESENT, self.ENTRY)
+            self.assertEqual([(v.field, v.value, v.apply) for v in verdicts],
+                             [("required", True, True), ("hle_fallback", False, True)])
 
-    def test_inconclusive_is_queued(self) -> None:
-        (verdict,) = rules.rule_bios_need(rules.Boot(rules.INCONCLUSIVE, "x"), self.ENTRY, self.RID)
-        self.assertFalse(verdict.apply)
+    def test_unknown_or_no_boot_alone_settles_nothing(self) -> None:
+        for none, present in ((run(rules.UNKNOWN), PRESENT), (NONE_HANGS, run(rules.HANG)),
+                              (NONE_HANGS, run(rules.UNKNOWN))):
+            (verdict,) = rules.rule_need(none, present, self.ENTRY)
+            self.assertFalse(verdict.apply)
+            self.assertEqual(verdict.rule, "unsettled")
+
+    def test_file_use(self) -> None:
+        self.assertEqual(rules.file_use(run(rules.BIOS_ERROR), NONE_HANGS, PRESENT), rules.REFUSED)
+        # It boots without the BIOS screens the right file shows: a fallback.
+        self.assertEqual(rules.file_use(run(rules.BOOTS, stages=("game",)), NONE_BOOTS, PRESENT),
+                         rules.REFUSED)
+        self.assertEqual(rules.file_use(run(rules.BOOTS, stages=("bios", "game")), NONE_BOOTS, PRESENT),
+                         rules.USED)
+        self.assertEqual(rules.file_use(run(rules.HANG), NONE_BOOTS, PRESENT), rules.USED)
+        self.assertEqual(rules.file_use(run(rules.CRASH), NONE_BOOTS, PRESENT), rules.USED)
+        # A hang the run with no file shows too says nothing about the file.
+        self.assertIsNone(rules.file_use(run(rules.HANG), NONE_HANGS, PRESENT))
+        self.assertIsNone(rules.file_use(run(rules.UNKNOWN), NONE_BOOTS, PRESENT))
+        self.assertIsNone(rules.file_use(run(rules.BIOS_ERROR), NONE_BOOTS, run(rules.HANG)))
+        no_logo = run(rules.BOOTS, stages=("game",))
+        self.assertIsNone(rules.file_use(run(rules.BOOTS, stages=("game",)), NONE_BOOTS, no_logo))
+
+    def test_validation_agrees_or_adds_size(self) -> None:
+        refused = run(rules.BOOTS, stages=("game",))
+        used = run(rules.HANG)
+        entry = {"name": "bios.bin", "validation": ["size", "md5"]}
+        verdict = rules.rule_validation(refused, refused, NONE_BOOTS, PRESENT, entry)
+        self.assertEqual((verdict.rule, verdict.apply), ("validation_agrees", True))
+        self.assertTrue(rules.agrees(entry, verdict))
+        added = rules.rule_validation(used, refused, NONE_BOOTS, PRESENT, {"name": "bios.bin"})
+        self.assertEqual((added.rule, added.value, added.apply), ("validation_size_added", ["size"], True))
+        self.assertFalse(rules.agrees({"name": "bios.bin"}, added))
+        self.assertEqual(rules.rule_validation(used, used, NONE_BOOTS, PRESENT, {"name": "b"}).rule,
+                         "validation_agrees")
+
+    def test_validation_contradicted_is_queued(self) -> None:
+        refused = run(rules.BIOS_ERROR)
+        used = run(rules.HANG)
+        cases = [
+            (refused, None, {"name": "b", "validation": ["size"]}),        # content check undeclared
+            (used, None, {"name": "b", "validation": ["md5"]}),            # declared, not done
+            (None, used, {"name": "b", "validation": ["size"]}),           # half of it runs
+            (used, refused, {"name": "b", "validation": {"core": ["md5"]}}),  # per-build form
+        ]
+        for zeros, half, entry in cases:
+            verdict = rules.rule_validation(zeros, half, NONE_BOOTS, PRESENT, entry)
+            self.assertEqual((verdict.rule, verdict.apply), ("validation_contradicted", False), entry)
+            self.assertFalse(rules.agrees(entry, verdict))
+        self.assertIsNone(rules.rule_validation(run(rules.UNKNOWN), None, NONE_BOOTS, PRESENT,
+                                                {"name": "b"}))
 
     def test_path_and_alias(self) -> None:
         self.assertIsNone(rules.rule_path(self.ENTRY, "bios.bin", rules.Opened("bios.bin", self.RID)))
@@ -204,8 +227,9 @@ class RuleTable(unittest.TestCase):
         self.assertEqual(verdict.rule, "never_opened")
 
     def test_variants(self) -> None:
-        self.assertIsNone(rules.rule_variants("same", ["a", "b"], self.ENTRY, self.RID))
-        self.assertFalse(rules.rule_variants("differs", ["b", "a"], self.ENTRY, self.RID).apply)
+        self.assertIsNone(rules.rule_variants([run(rules.BOOTS, "a"), run(rules.BOOTS, "b")], self.ENTRY))
+        verdict = rules.rule_variants([run(rules.BOOTS, "b"), run(rules.HANG, "a")], self.ENTRY)
+        self.assertEqual((verdict.value, verdict.apply), (["a", "b"], False))
 
     def test_agreement_and_evidence(self) -> None:
         self.assertTrue(rules.agrees({"name": "a"}, rules.Verdict("r", "hle_fallback", False, "", ())))
@@ -213,6 +237,22 @@ class RuleTable(unittest.TestCase):
         self.assertTrue(rules.agrees(self.ENTRY, rules.Verdict("r", "aliases", ["Bios.bin"], "", ())))
         merged = rules.merge_evidence({"required": ["x"]}, rules.Verdict("r", "required", True, "", ("y", "x")))
         self.assertEqual(merged, {"required": ["x", "y"]})
+        # An id a frame comparison wrote no longer proves the field.
+        merged = rules.merge_evidence({"required": ["old"], "path": ["p"]},
+                                      rules.Verdict("r", "required", True, "", ("new",)),
+                                      keep=lambda rid: rid != "old")
+        self.assertEqual(merged, {"path": ["p"], "required": ["new"]})
+
+    def test_reference_candidates(self) -> None:
+        def rep(hashes: list[str]) -> dict:
+            return {"observables": {"frame_hashes": [[i * 10, h] for i, h in enumerate(hashes)],
+                                    "uniform_frames": ["black"]}}
+        record = {"variants": [{"repetitions": [
+            rep(["black"] * 4 + ["logo"] * 3 + ["flash"] + ["title"] * 5),
+            rep(["black"] * 5 + ["logo"] * 3 + ["title"] * 3)]}]}
+        self.assertEqual(campaign.reference_candidates(record),
+                         [{"sha1": "logo", "first_frame": 40, "samples": 3},
+                          {"sha1": "title", "first_frame": 80, "samples": 5}])
 
 
 class ProfileEditor(unittest.TestCase):
@@ -286,6 +326,11 @@ class FakeHost(campaign.Host):
         return result
 
 
+def screen(base: int) -> str:
+    """The sha1 of a frame of FAKE_CORE: eight 16-bit pixels from base up."""
+    return hashlib.sha1(struct.pack("<8H", *range(base, base + 8))).hexdigest()
+
+
 class Workspace(unittest.TestCase):
     def setUp(self) -> None:
         TMP.mkdir(parents=True, exist_ok=True)
@@ -309,9 +354,17 @@ class Workspace(unittest.TestCase):
                 "name": "Fake Game (World)", "catalog": "no-intro", "reason": "a test",
                 "frames": 12, "frame_hash_every": 2}]}}}), encoding="utf-8")
         self.game = self.tmp / "content" / "fakesys" / "Fake Game (World)"
+        self.oracle = self.tmp / "oracle.yml"
+        self.oracle.write_text(yaml.safe_dump({
+            "error_patterns": [], "cores": {}, "content": [{
+                "name": "Fake Game (World)", "sha1": hashlib.sha1(b"GAME").hexdigest(),
+                "references": {"Fake": {"recorded_from": "2026-10-27-000000000000", "frames": [
+                    {"sha1": screen(1), "stage": "bios", "shows": "the fake logo"},
+                    {"sha1": screen(20), "stage": "game", "shows": "the fake title"}]}}}]}),
+            encoding="utf-8")
         self.paths = campaign.Paths(root=self.root, home=self.tmp / "home",
                                     content=self.tmp / "content", db=self.root / "database.json",
-                                    content_file=self.content_file)
+                                    content_file=self.content_file, oracle=self.oracle)
 
     def tearDown(self) -> None:
         shutil.rmtree(self.tmp, ignore_errors=True)
@@ -333,8 +386,12 @@ class Workspace(unittest.TestCase):
 class StateAndResume(Workspace):
     def test_should_run(self) -> None:
         self.assertTrue(campaign.should_run(None, "a", "d"))
-        done = {"status": "done", "core_sha256": "a", "profile_digest": "d"}
+        done = {"status": "done", "core_sha256": "a", "profile_digest": "d",
+                "record_version": mr.RECORD_VERSION, "oracle_sha256": ""}
         self.assertFalse(campaign.should_run(done, "a", "d"))
+        # A record from before the verdicts, or judged against another oracle, runs again.
+        self.assertTrue(campaign.should_run({**done, "record_version": 3}, "a", "d"))
+        self.assertTrue(campaign.should_run(done, "a", "d", "f" * 64))
         self.assertTrue(campaign.should_run(done, "b", "d"))
         self.assertTrue(campaign.should_run(done, "a", "e"))
         self.assertTrue(campaign.should_run({**done, "status": "pending"}, "a", "d"))
@@ -752,8 +809,14 @@ class Findings(Workspace):
     def entry(self, index: int = 0) -> dict:
         return yaml.safe_load(self.profile.read_text())["files"][index]
 
+    def none(self, verdict: str) -> rules.Run:
+        return rules.Run(verdict, self.RID, "none")
+
+    def present(self) -> rules.Run:
+        return rules.Run(rules.BOOTS, self.RID, "bios.bin", ("game",))
+
     def test_a_settled_field_is_written_with_its_evidence(self) -> None:
-        verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
+        verdicts = rules.rule_need(self.none(rules.BOOTS), self.present(), self.entry())
         self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
         entry = self.entry()
         self.assertEqual(entry["hle_fallback"], True)
@@ -767,7 +830,7 @@ class Findings(Workspace):
         self.assertEqual(len(self.lines()), 2)
 
     def test_an_agreeing_run_adds_evidence_only(self) -> None:
-        verdicts = rules.rule_bios_need(rules.Boot(rules.FAILS, "nothing"), self.entry(), self.RID)
+        verdicts = rules.rule_need(self.none(rules.HANG), self.present(), self.entry())
         self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
         entry = self.entry()
         self.assertEqual((entry["required"], entry["hle_fallback"]), (True, False))
@@ -775,7 +838,7 @@ class Findings(Workspace):
         self.assertEqual(self.lines(), [])
 
     def test_a_later_agreement_resolves_the_open_finding(self) -> None:
-        (verdict,) = rules.rule_bios_need(rules.Boot(rules.INCONCLUSIVE, "?"), self.entry(), self.RID)
+        (verdict,) = rules.rule_need(self.none(rules.UNKNOWN), self.present(), self.entry())
         self.c.settle("fake", self.content, self.core, 0, self.entry(), [verdict])
         self.assertEqual(self.lines()[-1]["status"], "open")
         later = rules.Verdict("inconclusive", "hle_fallback", False, "agrees now", ("2026-10-28-000000000000",))
@@ -789,7 +852,7 @@ class Findings(Workspace):
         self.assertNotIn("evidence", self.entry(1))
 
     def test_finding_lines_validate(self) -> None:
-        verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
+        verdicts = rules.rule_need(self.none(rules.BOOTS), self.present(), self.entry())
         self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
         self.c.write_run_line(self.c.current)
         try:
@@ -802,7 +865,7 @@ class Findings(Workspace):
                 self.assertEqual(list(Draft202012Validator(schema).iter_errors(line)), [], name)
 
     def test_the_batch_commit_plan(self) -> None:
-        verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
+        verdicts = rules.rule_need(self.none(rules.BOOTS), self.present(), self.entry())
         self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
         self.c.current.touched.add(mr.records_path("fake"))
         plan = self.c.plan_commits(set())
@@ -937,12 +1000,19 @@ class FakeCorePasses(Workspace):
         isolations = {rep["isolation"] for r in records for v in r["variants"] for rep in v["repetitions"]}
         self.assertEqual(isolations, {campaign.core_probe.SANDBOX})
         kinds = sorted(r["subject"].split("-")[0] for r in records)
-        self.assertEqual(kinds, ["alias", "bench", "boot", "files"])
-        boot = next(r for r in records if r["subject"].startswith("boot"))
-        alias = next(r for r in records if r["subject"].startswith("alias"))
+        self.assertEqual(kinds, ["alias", "files", "negative", "none"])
+        by_kind = {r["subject"].split("-")[0]: r for r in records}
+        self.assertEqual(by_kind["files"]["variants"][0]["verdict"], "boots")
+        self.assertEqual(by_kind["none"]["variants"][0]["verdict_evidence"]["stages"], ["game"])
+        self.assertEqual(by_kind["negative"]["check"]["result"]["verdicts"],
+                         {"bios.bin": "boots", "bios.bin-zeros-4": "hang",
+                          "bios.bin-head-2-of-bios.bin": "boots"})
         entry = yaml.safe_load((self.root / "emulators" / "fake.yml").read_text())["files"][0]
         self.assertEqual(entry["hle_fallback"], True)
-        self.assertEqual(entry["evidence"], {"aliases": [alias["id"]], "hle_fallback": [boot["id"]]})
+        self.assertEqual(entry["validation"], ["size"])
+        self.assertEqual(entry["evidence"], {
+            "aliases": [by_kind["alias"]["id"]], "hle_fallback": [by_kind["none"]["id"]],
+            "validation": [by_kind["none"]["id"], by_kind["negative"]["id"]]})
         statuses = {(f["field"], f["status"]) for f in mr.read_campaign(self.root, mr.FINDINGS)}
         self.assertEqual(statuses, {("hle_fallback", "applied"), ("required", "open"),
                                     ("aliases", "applied")})
@@ -961,9 +1031,23 @@ class FakeCorePasses(Workspace):
         with rebuilt.open("ab") as handle:
             handle.write(b"\0")
         third = self.passing("hle", core=rebuilt)
-        # BIOS.BIN is declared now, so no alias run follows the boot test.
+        # BIOS.BIN is declared now, so no alias run follows.
         self.assertEqual(len(third.run(set(), None).records), 3)
 
+    def test_a_size_check_the_profile_lacks_is_written(self) -> None:
+        profile = self.root / "emulators" / "fake.yml"
+        profile.write_text(PROFILE.replace("    validation: [size]\n", ""), encoding="utf-8")
+        self.passing("hle").run(set(), None)
+        entry = yaml.safe_load(profile.read_text())["files"][0]
+        self.assertEqual(entry["validation"], ["size"])
+        applied = {f["field"] for f in mr.read_campaign(self.root, mr.FINDINGS) if f["status"] == "applied"}
+        self.assertIn("validation", applied)
+
+    def test_bench_runs_only_when_asked(self) -> None:
+        self.passing("plain", bench=True).run(set(), None)
+        subjects = [s.record["subject"] for s in mr.read_records(self.root)]
+        self.assertEqual(sum(s.startswith("bench") for s in subjects), 1)
+
     def test_no_stand_in_settles_required_and_finds_an_alias(self) -> None:
         c = self.passing()
         c.run(set(), None)
@@ -982,9 +1066,13 @@ class FakeCorePasses(Workspace):
         self.assertEqual(left.stdout, b"")
         checks = c.state["checks"]["fake"]
         self.assertTrue(checks)
-        self.assertEqual({entry["status"] for entry in checks.values()}, {"failed"})
-        self.assertEqual({entry["reason"] for entry in checks.values()}, {"timeout"})
-        self.assertEqual(mr.read_records(self.root), [])
+        # A run killed after its timeout is a record whose verdict is a hang.
+        self.assertEqual({entry["status"] for entry in checks.values()}, {"done"})
+        records = [s.record for s in mr.read_records(self.root)]
+        self.assertTrue(records)
+        self.assertEqual({v["verdict"] for r in records for v in r["variants"]}, {"hang"})
+        self.assertEqual({rep["verdict_evidence"]["rule"] for r in records for v in r["variants"]
+                          for rep in v["repetitions"]}, {"timeout"})
 
     def test_missing_content_leaves_the_checks_pending(self) -> None:
         c = self.passing(romfetch=str(self.tmp / "no-romfetch"))

tests/test_core_probe.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import argparse
 import contextlib
+import hashlib
 import io
 import json
 import os
@@ -20,6 +21,7 @@ import tempfile
 import unittest
 import unittest.mock
 from pathlib import Path
+from typing import ClassVar
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT / "scripts"))
@@ -42,6 +44,7 @@ FAKE_CORE = r"""
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <signal.h>
 struct retro_system_info { const char *n, *v, *e; bool full, block; };
 struct retro_game_geometry { unsigned bw, bh, mw, mh; float a; };
 struct retro_system_timing { double fps, rate; };
@@ -78,11 +81,15 @@ bool retro_load_game(const void *g) {
   if (env(27, &cb)) {
     logger = cb;
     cb.log(1, "card %s byte %d\n", f ? "present" : "absent", (int)card);
-    cb.log(1, "tag %s\n", tag ? tag : "unset"); }
+    cb.log(1, "tag %s\n", tag ? tag : "unset");
+    if (card == 0xEE) cb.log(3, "BIOS not found: card.bin\n"); }
   return true; }
 void retro_unload_game(void) {}
 void retro_run(void) {
   int16_t s[4] = {1, 2, 3, 4};
+  const char *end = getenv("CORE_PROBE_TEST_END");
+  if (end && frames == 1 && !strcmp(end, "crash")) { if (logger.log) logger.log(3, "about to fall\n"); raise(SIGSEGV); }
+  if (end && frames == 1 && !strcmp(end, "hang")) for (;;) {}
   if (frames++ == 0 && logger.log) logger.log(1, "ports %u\n", ports);
   input(0, 1, 0, 8);
   for (int i = 0; i < 8; i++) pixels[i] = card;
@@ -177,6 +184,127 @@ class ParsingTests(unittest.TestCase):
             self.assertEqual([p.name for p in core_probe.cue_tracks(cue)], ["a b.bin", "c.bin"])
 
 
+def frames(*pairs: tuple[int, str]) -> list[list]:
+    return [[frame, digest] for frame, digest in pairs]
+
+
+class OracleTests(unittest.TestCase):
+    """judge() turns a repetition into a verdict from positive evidence only."""
+
+    TITLE, LOGO, ERROR, BLACK, OTHER = ("1" * 40, "2" * 40, "3" * 40, "4" * 40, "5" * 40)
+    ORACLE: ClassVar[dict] = {"references": [{"sha1": "2" * 40, "stage": "bios", "shows": "the console logo"},
+                             {"sha1": "1" * 40, "stage": "game", "shows": "the title"}],
+              "error_patterns": [r"BIOS not found"],
+              "error_screens": [{"sha1": "3" * 40, "shows": "the core's missing-BIOS message"}]}
+
+    def rep(self, hashes: list, logs: list[str] = (), uniform: list[str] = (), loaded: bool = True,
+            **extra: object) -> dict:
+        return {"loaded": loaded, "library_name": "Fake", "frames": 100, "warmup": 0,
+                "logs": [{"frame": 0, "level": "error", "text": text} for text in logs],
+                "observables": {"frame_hashes": hashes, "uniform_frames": list(uniform)}, **extra}
+
+    def test_a_reference_frame_is_a_boot_with_its_stages(self) -> None:
+        verdict, evidence = core_probe.judge(
+            self.rep(frames((0, self.BLACK), (10, self.LOGO), (20, self.TITLE)),
+                     logs=["BIOS not found"]), self.ORACLE)
+        self.assertEqual(verdict, "boots")
+        self.assertEqual((evidence["rule"], evidence["frame"], evidence["stages"]),
+                         ("reference_frame", 10, ["bios", "game"]))
+
+    def test_an_error_pattern_or_screen_without_a_reference_is_a_bios_error(self) -> None:
+        verdict, evidence = core_probe.judge(
+            self.rep(frames((0, self.OTHER)), logs=["x", "BIOS not found: scph5501.bin"]), self.ORACLE)
+        self.assertEqual((verdict, evidence["rule"], evidence["log"]),
+                         ("bios_error", "error_pattern", "BIOS not found: scph5501.bin"))
+        verdict, evidence = core_probe.judge(self.rep(frames((0, self.ERROR))), self.ORACLE)
+        self.assertEqual((verdict, evidence["rule"], evidence["frame"]), ("bios_error", "error_screen", 0))
+        self.assertEqual(core_probe.judge(self.rep([], logs=["BIOS not found"], loaded=False),
+                                          self.ORACLE)[0], "bios_error")
+
+    def test_process_status_decides_crash_and_hang_first(self) -> None:
+        crashed = self.rep(frames((0, self.TITLE)), exit={"status": -11, "signal": "SIGSEGV"})
+        self.assertEqual(core_probe.judge(crashed, self.ORACLE)[0], "crash")
+        killed = self.rep([], exit={"status": None, "signal": "SIGKILL", "timeout_s": 5})
+        self.assertEqual(core_probe.judge(killed, self.ORACLE), (
+            "hang", {"rule": "timeout", "detail": "killed after 5 s", "exit_status": None,
+                     "signal": "SIGKILL"}))
+
+    def test_a_one_colour_second_half_is_a_hang(self) -> None:
+        run = self.rep(frames((0, self.OTHER), (10, self.OTHER), (20, self.BLACK), (30, self.BLACK)),
+                       uniform=[self.BLACK])
+        self.assertEqual(core_probe.judge(run, self.ORACLE)[1]["rule"], "uniform_tail")
+        lit = self.rep(frames((0, self.BLACK), (10, self.BLACK), (20, self.OTHER), (30, self.BLACK)),
+                       uniform=[self.BLACK])
+        self.assertEqual(core_probe.judge(lit, self.ORACLE)[0], "unknown")
+
+    def test_frames_that_differ_are_never_a_boot(self) -> None:
+        verdict, evidence = core_probe.judge(self.rep(frames((0, self.OTHER), (10, "6" * 40))),
+                                             self.ORACLE)
+        self.assertEqual((verdict, evidence["rule"]), ("unknown", "no_evidence"))
+        verdict, evidence = core_probe.judge(self.rep(frames((0, self.TITLE))),
+                                             {**self.ORACLE, "references": []})
+        self.assertEqual(verdict, "unknown")
+        self.assertIn("no reference frame is known", evidence["detail"])
+        self.assertEqual(core_probe.judge(self.rep([], loaded=False), self.ORACLE)[0], "unknown")
+
+    def test_a_variant_needs_every_repetition_to_agree(self) -> None:
+        boot = {"verdict": "boots", "verdict_evidence": {"rule": "reference_frame", "detail": "d",
+                                                         "frame": 3, "stages": ["game"]}}
+        verdict, evidence = core_probe.variant_verdict([boot, boot])
+        self.assertEqual((verdict, evidence["rule"], evidence["stages"]),
+                         ("boots", "repetitions_agree", ["game"]))
+        hang = {"verdict": "hang", "verdict_evidence": {"rule": "uniform_tail", "detail": "d"}}
+        verdict, evidence = core_probe.variant_verdict([boot, hang])
+        self.assertEqual((verdict, evidence["repetitions"]), ("unknown", ["boots", "hang"]))
+
+    def test_oracle_entries_are_picked_by_content_and_core(self) -> None:
+        data = {"error_patterns": [{"pattern": "shared"}],
+                "cores": {"Fake": {"error_patterns": [{"pattern": "own"}],
+                                   "error_screens": [{"sha1": self.ERROR, "shows": "s"}]}},
+                "content": [{"name": "disc", "sha1": "a" * 40, "references": {
+                    "Fake": {"frames": [{"sha1": self.TITLE, "stage": "game", "shows": "t"}]},
+                    "Other": {"frames": [{"sha1": self.LOGO, "stage": "bios", "shows": "l"}]}}}]}
+        picked = core_probe.oracle_for(data, "a" * 40, "Fake")
+        self.assertEqual(picked["references"], [{"sha1": self.TITLE, "stage": "game", "shows": "t"}])
+        self.assertEqual(picked["error_patterns"], ["shared", "own"])
+        self.assertEqual(core_probe.oracle_for(data, "b" * 40, "Fake")["references"], [])
+        self.assertEqual(core_probe.oracle_for(data, "a" * 40, "Nobody")["error_patterns"], ["shared"])
+
+    def test_the_repository_oracle_is_well_formed(self) -> None:
+        import yaml
+        data = yaml.safe_load(core_probe.DEFAULT_ORACLE.read_text(encoding="utf-8"))
+        patterns = list(data.get("error_patterns") or ())
+        for core in (data.get("cores") or {}).values():
+            patterns += core.get("error_patterns") or []
+            for screen in core.get("error_screens") or ():
+                self.assertRegex(screen["sha1"], r"^[0-9a-f]{40}$")
+        for pattern in patterns:
+            self.assertTrue(pattern.get("source_ref") or pattern.get("reason"), pattern)
+            core_probe.re.compile(pattern["pattern"])
+        for item in data.get("content") or ():
+            self.assertRegex(item["sha1"], r"^[0-9a-f]{40}$")
+            for block in item["references"].values():
+                self.assertRegex(block["recorded_from"], r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$")
+                stages = {frame["stage"] for frame in block["frames"]}
+                self.assertIn("game", stages, item["name"])
+                for frame in block["frames"]:
+                    self.assertRegex(frame["sha1"], r"^[0-9a-f]{40}$")
+                    self.assertIn(frame["stage"], ("bios", "game"))
+                    self.assertTrue(frame["shows"])
+
+    def test_made_files(self) -> None:
+        zeros = core_probe.Synthetic.parse("@zeros:16")
+        self.assertEqual((zeros.label("bios.bin"), zeros.source), ("bios.bin-zeros-16", None))
+        head = core_probe.Synthetic.parse("@head:8:/x/scph5501.bin")
+        self.assertEqual(head.label("bios.bin"), "bios.bin-head-8-of-scph5501.bin")
+        for bad in ("@zeros:", "@zeros:8:/x", "@head:8", "@ones:8", "@head:x:/y"):
+            with self.assertRaises(argparse.ArgumentTypeError):
+                core_probe.Synthetic.parse(bad)
+        self.assertEqual(core_probe.exit_description(-11), {"status": -11, "signal": "SIGSEGV"})
+        self.assertEqual(core_probe.exit_description(139), {"status": 139, "signal": "SIGSEGV"})
+        self.assertEqual(core_probe.exit_description(1), {"status": 1, "signal": None})
+
+
 class ResultTests(unittest.TestCase):
     @staticmethod
     def variant(label: str, value: str, loaded: bool = True) -> dict:
@@ -290,6 +418,76 @@ class FakeCoreTests(unittest.TestCase):
         schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text())
         self.assertEqual(list(Draft202012Validator(schema).iter_errors(line)), [])
 
+    def oracle_file(self) -> Path:
+        title = hashlib.sha1(struct.pack("<8H", *[1] * 8)).hexdigest()
+        content = self.tmp / "game.bin"
+        content.write_bytes(b"game")
+        path = self.tmp / "oracle.yml"
+        path.write_text(
+            "error_patterns:\n  - pattern: 'BIOS not found'\n    reason: test\n"
+            "cores:\n  Fake:\n    error_patterns: []\n    error_screens: []\n"
+            f"content:\n  - name: game\n    sha1: {core_probe.file_digest(content)}\n"
+            "    references:\n      Fake:\n        recorded_from: 2026-10-27-000000000000\n"
+            f"        frames:\n          - sha1: {title}\n            stage: game\n"
+            "            shows: card one\n", encoding="utf-8")
+        return path
+
+    def test_verdicts_of_a_negative_check(self) -> None:
+        oracle = self.oracle_file()
+        error = self.tmp / "error.bin"
+        error.write_bytes(b"\xee")
+        code, record = self.probe(["check", "--core", str(self.core), "--frames", "4",
+                                   "--frame-hash-every", "1", "--content", str(self.tmp / "game.bin"),
+                                   "--oracle", str(oracle), "--vary", "card.bin=",
+                                   "--vary", f"card.bin={self.cards[0]}", "--vary", f"card.bin={error}",
+                                   "--vary", "card.bin=@zeros:1",
+                                   "--vary", f"card.bin=@head:0:{self.cards[0]}",
+                                   "--observable", "frame_hashes", "--claim", "c", "--method", "m",
+                                   "--no-trace", "--repeat", "2"])
+        self.assertEqual(code, 0)
+        verdicts = record["check"]["result"]["verdicts"]
+        self.assertEqual(verdicts, {"card.bin-absent": "hang", "card0.bin": "boots",
+                                    "error.bin": "bios_error", "card.bin-zeros-1": "hang",
+                                    "card.bin-head-0-of-card0.bin": "hang"})
+        boot = record["variants"][1]
+        self.assertEqual((boot["verdict_evidence"]["rule"], boot["verdict_evidence"]["stages"]),
+                         ("repetitions_agree", ["game"]))
+        self.assertEqual(boot["repetitions"][0]["verdict_evidence"]["rule"], "reference_frame")
+        self.assertEqual(record["variants"][3]["system_files"],
+                         {"card.bin": hashlib.sha1(b"\x00").hexdigest()})
+        self.assertEqual(record["inputs"]["oracle"]["error_patterns"], ["BIOS not found"])
+        self.assertEqual(record["record_version"], 4)
+        self.assert_schema(record)
+
+    def test_a_crash_and_a_hang_are_verdicts(self) -> None:
+        oracle = self.oracle_file()
+        code, record = self.probe(["run", "--core", str(self.core), "--frames", "4",
+                                   "--frame-hash-every", "1", "--oracle", str(oracle),
+                                   "--env", "CORE_PROBE_TEST_END=crash"])
+        self.assertEqual(code, 0)
+        crashed = record["variants"][0]
+        self.assertEqual((crashed["verdict"], crashed["repetitions"][0]["exit"]["signal"]),
+                         ("crash", "SIGSEGV"))
+        self.assertEqual(crashed["repetitions"][0]["logs"][-1]["text"], "about to fall")
+        self.assertEqual(crashed["repetitions"][0]["observables"]["frame_hashes"][0][0], 0)
+        self.assert_schema(record)
+        code, record = self.probe(["run", "--core", str(self.core), "--frames", "4",
+                                   "--frame-hash-every", "1", "--oracle", str(oracle), "--timeout", "3",
+                                   "--env", "CORE_PROBE_TEST_END=hang", "--no-trace"])
+        self.assertEqual(code, 0)
+        self.assertEqual((record["variants"][0]["verdict"],
+                          record["variants"][0]["repetitions"][0]["verdict_evidence"]["rule"]),
+                         ("hang", "timeout"))
+
+    def assert_schema(self, record: dict) -> None:
+        try:
+            from jsonschema import Draft202012Validator
+        except ImportError:
+            return
+        schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text())
+        sealed = measurement_records.seal({**record, "emulator": "fake", "subject": "s"})
+        self.assertEqual([e.message for e in Draft202012Validator(schema).iter_errors(sealed)], [])
+
     def test_second_run_is_appended_beside_the_first(self) -> None:
         record_dir = checkout(self.tmp / "appended")
         argv = ["run", "--core", str(self.core), "--frames", "1", "--no-trace", "--max-load", "1000",