From aa6ee6db1cbade04189ddec3043bc69f1f712670 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:25:30 +0100 Subject: [PATCH] feat: add the boot verdict to the record schema Record version 4 carries verdict and verdict_evidence on every repetition and variant, verdicts on a check result, and inputs.oracle. Version 3 records stay valid. --- schemas/core-probe-record.schema.json | 260 +++++++++++++++++++++++++- scripts/measurement_records.py | 6 +- tests/test_measurements.py | 37 +++- 3 files changed, 295 insertions(+), 8 deletions(-) diff --git a/schemas/core-probe-record.schema.json b/schemas/core-probe-record.schema.json index 4fa4d1737..c57c54613 100644 --- a/schemas/core-probe-record.schema.json +++ b/schemas/core-probe-record.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-record.schema.json", "title": "RetroBIOS core probe record", - "description": "One line of measurements/.jsonl: one run of scripts/core_probe.py, a libretro core executed headless, in full, with what is needed to judge and reproduce it. Every repetition, its core logs and per-frame hashes, and the raw timing values are in the line.", + "description": "One line of measurements/.jsonl: one run of scripts/core_probe.py, a libretro core executed headless, in full, with what is needed to judge and reproduce it. Every repetition, its core logs and per-frame hashes, and the raw timing values are in the line. From record_version 4 every repetition and every variant carries the boot oracle's verdict (boots, bios_error, crash, hang, unknown) with the evidence that decided it.", "type": "object", "additionalProperties": false, "required": [ @@ -22,7 +22,11 @@ ], "properties": { "record_version": { - "const": 3 + "enum": [ + 3, + 4 + ], + "description": "3: records before the boot oracle. 4: every repetition and variant carries `verdict` and `verdict_evidence`, and `inputs.oracle` names what the verdicts were judged against." }, "id": { "type": "string", @@ -353,6 +357,9 @@ "frame_hash_every": { "type": "integer", "minimum": 0 + }, + "oracle": { + "$ref": "#/$defs/oracle" } } }, @@ -398,6 +405,13 @@ "items": { "type": "string" } + }, + "verdict": { + "$ref": "#/$defs/verdict", + "description": "The verdict every repetition of the variant agrees on, else unknown." + }, + "verdict_evidence": { + "$ref": "#/$defs/verdict_evidence" } } } @@ -492,6 +506,13 @@ } } } + }, + "verdicts": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/verdict" + }, + "description": "Variant label -> the variant's verdict." } } } @@ -535,6 +556,51 @@ "check" ] } + }, + { + "if": { + "properties": { + "record_version": { + "const": 4 + } + } + }, + "then": { + "properties": { + "inputs": { + "required": [ + "oracle" + ] + }, + "variants": { + "items": { + "required": [ + "verdict", + "verdict_evidence" + ], + "properties": { + "repetitions": { + "items": { + "required": [ + "verdict", + "verdict_evidence" + ] + } + } + } + } + }, + "check": { + "properties": { + "result": { + "required": [ + "verdicts" + ] + } + } + } + } + } } ], "$defs": { @@ -747,8 +813,198 @@ } } ] + }, + "verdict": { + "$ref": "#/$defs/verdict" + }, + "verdict_evidence": { + "$ref": "#/$defs/verdict_evidence" + }, + "exit": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "signal" + ], + "properties": { + "status": { + "type": [ + "integer", + "null" + ] + }, + "signal": { + "type": [ + "string", + "null" + ] + }, + "timeout_s": { + "type": "number", + "minimum": 0 + } + }, + "description": "How a worker that did not finish ended: its exit status or the signal that killed it, or the timeout after which the probe killed it." } } + }, + "verdict": { + "enum": [ + "boots", + "bios_error", + "crash", + "hang", + "unknown" + ], + "description": "boots: a reference frame of the content was reached. bios_error: an error pattern of the core log or a known error screen, and no reference frame. crash: the worker died. hang: killed after the timeout, or every sampled frame of the second half is one colour. unknown: none of these." + }, + "verdict_evidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "rule", + "detail" + ], + "properties": { + "rule": { + "enum": [ + "reference_frame", + "error_pattern", + "error_screen", + "exit_status", + "timeout", + "uniform_tail", + "no_evidence", + "repetitions_agree", + "repetitions_disagree" + ] + }, + "detail": { + "type": "string", + "minLength": 1 + }, + "frame": { + "type": "integer", + "minimum": 0 + }, + "sha1": { + "$ref": "#/$defs/sha1" + }, + "stages": { + "type": "array", + "items": { + "$ref": "#/$defs/stage" + }, + "description": "Stages of the reference frames the run reached, in the order first reached." + }, + "pattern": { + "type": "string" + }, + "log": { + "type": "string" + }, + "exit_status": { + "type": [ + "integer", + "null" + ] + }, + "signal": { + "type": [ + "string", + "null" + ] + }, + "repetitions": { + "type": "array", + "items": { + "$ref": "#/$defs/verdict" + } + } + } + }, + "stage": { + "enum": [ + "bios", + "game" + ], + "description": "bios: a screen only the console's own BIOS draws (its logo). game: a screen of the content itself." + }, + "oracle": { + "oneOf": [ + { + "type": "null" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "file", + "sha256", + "references", + "error_patterns", + "error_screens" + ], + "properties": { + "file": { + "type": "string" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "references": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "sha1", + "stage", + "shows" + ], + "properties": { + "sha1": { + "$ref": "#/$defs/sha1" + }, + "stage": { + "$ref": "#/$defs/stage" + }, + "shows": { + "type": "string" + } + } + } + }, + "error_patterns": { + "type": "array", + "items": { + "type": "string" + } + }, + "error_screens": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "sha1", + "shows" + ], + "properties": { + "sha1": { + "$ref": "#/$defs/sha1" + }, + "shows": { + "type": "string" + } + } + } + } + } + } + ], + "description": "What the verdicts were judged against: the reference frames of this content and core, the core's error patterns and error screens, from the oracle file at its sha256. Null when the run had no oracle entry." } } } diff --git a/scripts/measurement_records.py b/scripts/measurement_records.py index d23db29b0..73e9c9e52 100644 --- a/scripts/measurement_records.py +++ b/scripts/measurement_records.py @@ -28,6 +28,8 @@ from dataclasses import dataclass from pathlib import Path RECORD_VERSION = 3 +# 3: before the boot oracle. 4: every repetition and variant carries a verdict. +RECORD_VERSIONS = (3, 4) RECORDS_DIR = "measurements" ID_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$") SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:[-_][a-z0-9]+)*$") @@ -137,8 +139,8 @@ def check_line(record: dict) -> list[str]: """The structural rules a line holds, without a JSON Schema validator.""" rid = record.get("id", "") problems = [] - if record.get("record_version") != RECORD_VERSION: - problems.append(f"{rid}: record_version {record.get('record_version')} != {RECORD_VERSION}") + if record.get("record_version") not in RECORD_VERSIONS: + problems.append(f"{rid}: record_version {record.get('record_version')} not in {RECORD_VERSIONS}") if not ID_PATTERN.match(str(rid)): problems.append(f"{rid!r}: not an id of the form YYYY-MM-DD-<12 hex>") elif str(record.get("recorded_at", ""))[:10] != rid[:10]: diff --git a/tests/test_measurements.py b/tests/test_measurements.py index dba5aadcd..4ea69407a 100644 --- a/tests/test_measurements.py +++ b/tests/test_measurements.py @@ -43,16 +43,21 @@ def repetition(value: str, frames: int = 3) -> dict: "last_frame_size": [4, 2], "tracing": "strace", "input_queries": 3, "input_pressed": 1, "files_opened": [{"path": "card.bin", "sha1_after": SHA1, "sha1_staged": SHA1}], "files_probed_absent": [], + "verdict": "boots", + "verdict_evidence": {"rule": "reference_frame", "detail": "reached the title", "frame": 150, + "sha1": value, "stages": ["game"]}, } def full_record(emulator: str, subject: str, when: str, kind: str = "check", values: tuple[str, ...] = ("1" * 40, "2" * 40), observable: str = "video_sha1") -> dict: variants = [{"label": f"v{i}", "system_files": {"card.bin": SHA1}, - "repetitions": [repetition(v), repetition(v)], "flags": []} + "repetitions": [repetition(v), repetition(v)], "flags": [], "verdict": "boots", + "verdict_evidence": {"rule": "repetitions_agree", "detail": "both boot", + "repetitions": ["boots", "boots"]}} for i, v in enumerate(values)] record = { - "record_version": mr.RECORD_VERSION, "emulator": emulator, "subject": subject, + "record_version": 4, "emulator": emulator, "subject": subject, "supersedes": None, "kind": kind, "recorded_at": when, "core": {"file": "fake_libretro.so", "sha256": SHA256, "repo": "https://example.org/fake", "commit": "c" * 40, "build_command": "make", "compiler": ["GCC 13"], @@ -67,7 +72,10 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check", "options": {"fake_option": "a"}, "env": {}, "frames": 3, "warmup": 0, "repetitions": 2 if kind == "check" else 5, "presses": [], "observables": ["frame_sha1", "video_sha1", "audio_sha1", "state_sha1", "frame_hashes"], - "frame_hash_every": 150, "joypad_ports": 2}, + "frame_hash_every": 150, "joypad_ports": 2, + "oracle": {"file": "tests/campaign/oracle.yml", "sha256": SHA256, + "references": [{"sha1": values[0], "stage": "game", "shows": "title"}], + "error_patterns": ["BIOS not found"], "error_screens": []}}, "variants": variants, "flags": [], } @@ -81,7 +89,8 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check", "observable": observable, "variable": "card.bin", "result": {"status": "differs" if len(groups) > 1 else "same", "statement": f"{observable} takes {len(groups)} values", - "groups": [{"value": json.loads(k), "variants": v} for k, v in groups.items()]}, + "groups": [{"value": json.loads(k), "variants": v} for k, v in groups.items()], + "verdicts": {v["label"]: v["verdict"] for v in variants}}, } else: record["variants"] = record["variants"][:1] @@ -304,6 +313,26 @@ class ReadAndIndex(Workspace): self.assertTrue(any("measurements/beta.jsonl:1" in e and "kind" in e for e in errors), errors) self.assertIsNotNone(Draft202012Validator) + def test_a_version_4_record_needs_its_verdicts(self) -> None: + try: + from jsonschema import Draft202012Validator # noqa: F401 + except ImportError: + self.skipTest("jsonschema is not installed") + import validate_schemas # noqa: PLC0415 + + lines = self.tree / mr.records_path("beta") + record = json.loads(lines.read_text()) + del record["variants"][0]["verdict"] + del record["variants"][0]["repetitions"][0]["verdict_evidence"] + lines.write_text(mr.line_text(record) + "\n") + errors = validate_schemas._measurement_errors(self.tree) + self.assertTrue(any("'verdict' is a required property" in e for e in errors), errors) + self.assertTrue(any("'verdict_evidence' is a required property" in e for e in errors), errors) + old = {**record, "record_version": 3} + lines.write_text(mr.line_text(old) + "\n") + self.assertFalse(any("required property" in e + for e in validate_schemas._measurement_errors(self.tree))) + class AppendOnly(Workspace): def setUp(self) -> None: