feat: judge recorded runs again offline

core_probe rejudge appends, for each live record the oracle now judges otherwise, a record with the same runs judged again: verdicts, bios_used and inputs.oracle recomputed, supersedes the earlier one, judged_at and judged_by beside recorded_at. Records of another probe are left to it. The campaign reads a superseded record as its successor, and a file the log shows refused or loaded settles validation with no BIOS screen.
This commit is contained in:
Abdessamad Derraz committed 2026-10-28 05:22:06 +01:00
1 parent 05192f43c7
commit ad7dc0ea17
5 files changed
+166 -8

No files matched your search

+23
View File
@@ -192,6 +192,20 @@ class RuleTable(unittest.TestCase):
no_logo = run(rules.BOOTS, stages=("game",))
self.assertIsNone(rules.file_use(run(rules.BOOTS, stages=("game",)), NONE_BOOTS, no_logo))
def test_the_bios_the_log_names_settles_the_file_use(self) -> None:
present = run(rules.BOOTS, stages=("game",)) # no BIOS screen to compare with
fallback = rules.Run(rules.BOOTS, "r", "zeros", ("game",), "builtin")
loaded = rules.Run(rules.BOOTS, "r", "other", ("game",), "staged:bios.bin")
self.assertEqual(rules.file_use(fallback, NONE_BOOTS, present), rules.REFUSED)
self.assertEqual(rules.file_use(loaded, NONE_BOOTS, present), rules.USED)
self.assertIsNone(rules.file_use(run(rules.BOOTS, stages=("game",)), NONE_BOOTS, present))
record = {"id": "x", "variants": [{"label": "a", "verdict": "boots", "bios_used": "builtin",
"verdict_evidence": {"rule": "r", "detail": "d"}}]}
self.assertEqual(rules.run_of(record, "a").bios_used, "builtin")
builtin_none = rules.Run(rules.BOOTS, "n", "none", ("game",), "builtin")
(hle,) = rules.rule_need(builtin_none, PRESENT, {"name": "a"})
self.assertIn("on its built-in BIOS", hle.statement)
def test_validation_agrees_or_adds_size(self) -> None:
refused = run(rules.BOOTS, stages=("game",))
used = run(rules.HANG)
@@ -411,6 +425,15 @@ class StateAndResume(Workspace):
self.assertTrue(campaign.should_run({**done, "status": "pending"}, "a", "d"))
self.assertFalse(campaign.should_run({**done, "status": "failed"}, "a", "d"))
def test_a_record_judged_again_is_read_in_its_place(self) -> None:
lines = self.root / mr.records_path("fake")
lines.parent.mkdir(parents=True, exist_ok=True)
old = {"id": "2026-10-27-aaaaaaaaaaaa", "supersedes": None, "verdict": "old"}
new = {"id": "2026-10-28-bbbbbbbbbbbb", "supersedes": old["id"], "verdict": "new"}
lines.write_text("\n".join(json.dumps(r) for r in (old, new)) + "\n", encoding="utf-8")
self.assertEqual(self.make().read_record("fake", old["id"])["verdict"], "new")
self.assertIsNone(self.make().read_record("fake", "2026-10-27-cccccccccccc"))
def test_only_its_own_oracle_entries_make_a_check_run_again(self) -> None:
key = [hashlib.sha1(b"GAME").hexdigest(), "Fake"]
before = self.make().oracle_digest(key)
+29
View File
@@ -565,6 +565,35 @@ class FakeCoreTests(unittest.TestCase):
self.assertEqual({v["bios_used"] for v in record["variants"]}, {"unknown"})
self.assert_schema(record)
def test_recorded_runs_are_judged_again_offline(self) -> None:
oracle = self.oracle_file()
record_dir = checkout(self.tmp / "rejudge")
code, first = self.probe(["run", "--core", str(self.core), "--frames", "4", "--frame-hash-every", "1",
"--content", str(self.tmp / "game.bin"), "--oracle", str(oracle),
"--no-trace", "--record", str(record_dir), "--max-load", "1000"])
self.assertEqual((code, first["variants"][0]["bios_used"]), (0, "unknown"))
# The core's fallback line becomes known: the same runs now name the BIOS.
oracle.write_text(oracle.read_text().replace(
" error_patterns: []\n", " error_patterns: []\n fallback_patterns:\n"
" - pattern: '^card absent'\n reason: test\n", 1), encoding="utf-8")
argv = ["rejudge", "--record", str(record_dir), "--oracle", str(oracle)]
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(core_probe.main(argv), 0)
lines = [json.loads(line) for line in (record_dir / "measurements" / "fake.jsonl").read_text().splitlines()]
self.assertEqual(len(lines), 2)
again = lines[1]
self.assertEqual((again["supersedes"], again["record_version"]), (first["id"], 5))
self.assertEqual(again["variants"][0]["bios_used"], "builtin")
self.assertEqual(again["variants"][0]["repetitions"][0]["logs"], first["variants"][0]["repetitions"][0]["logs"])
self.assertEqual(again["recorded_at"], first["recorded_at"])
self.assertIn("judged_at", again)
self.assertEqual(measurement_records.check_line(again), [])
self.assert_schema(again)
# Judged once: a second pass has nothing to add.
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
core_probe.main(argv)
self.assertEqual(len((record_dir / "measurements" / "fake.jsonl").read_text().splitlines()), 2)
def test_a_crash_and_a_hang_are_verdicts(self) -> None:
oracle = self.oracle_file()
code, record = self.probe(["run", "--core", str(self.core), "--frames", "4",