mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 00:27:20 -05:00
feat: add the boot verdict to the record schema
Record version 4 carries verdict and verdict_evidence on every repetition and variant, verdicts on a check result, and inputs.oracle. Version 3 records stay valid.
This commit is contained in:
Abdessamad Derraz committed
1 parent
dcaf9d4764
commit
aa6ee6db1c
3 files changed
+295
-8
No files matched your search
@@ -2,7 +2,7 @@
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-record.schema.json",
|
||||
"title": "RetroBIOS core probe record",
|
||||
"description": "One line of measurements/<emulator>.jsonl: one run of scripts/core_probe.py, a libretro core executed headless, in full, with what is needed to judge and reproduce it. Every repetition, its core logs and per-frame hashes, and the raw timing values are in the line.",
|
||||
"description": "One line of measurements/<emulator>.jsonl: one run of scripts/core_probe.py, a libretro core executed headless, in full, with what is needed to judge and reproduce it. Every repetition, its core logs and per-frame hashes, and the raw timing values are in the line. From record_version 4 every repetition and every variant carries the boot oracle's verdict (boots, bios_error, crash, hang, unknown) with the evidence that decided it.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
@@ -22,7 +22,11 @@
|
||||
],
|
||||
"properties": {
|
||||
"record_version": {
|
||||
"const": 3
|
||||
"enum": [
|
||||
3,
|
||||
4
|
||||
],
|
||||
"description": "3: records before the boot oracle. 4: every repetition and variant carries `verdict` and `verdict_evidence`, and `inputs.oracle` names what the verdicts were judged against."
|
||||
},
|
||||
"id": {
|
||||
"type": "string",
|
||||
@@ -353,6 +357,9 @@
|
||||
"frame_hash_every": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"oracle": {
|
||||
"$ref": "#/$defs/oracle"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -398,6 +405,13 @@
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"verdict": {
|
||||
"$ref": "#/$defs/verdict",
|
||||
"description": "The verdict every repetition of the variant agrees on, else unknown."
|
||||
},
|
||||
"verdict_evidence": {
|
||||
"$ref": "#/$defs/verdict_evidence"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -492,6 +506,13 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"verdicts": {
|
||||
"type": "object",
|
||||
"additionalProperties": {
|
||||
"$ref": "#/$defs/verdict"
|
||||
},
|
||||
"description": "Variant label -> the variant's verdict."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -535,6 +556,51 @@
|
||||
"check"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"if": {
|
||||
"properties": {
|
||||
"record_version": {
|
||||
"const": 4
|
||||
}
|
||||
}
|
||||
},
|
||||
"then": {
|
||||
"properties": {
|
||||
"inputs": {
|
||||
"required": [
|
||||
"oracle"
|
||||
]
|
||||
},
|
||||
"variants": {
|
||||
"items": {
|
||||
"required": [
|
||||
"verdict",
|
||||
"verdict_evidence"
|
||||
],
|
||||
"properties": {
|
||||
"repetitions": {
|
||||
"items": {
|
||||
"required": [
|
||||
"verdict",
|
||||
"verdict_evidence"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"check": {
|
||||
"properties": {
|
||||
"result": {
|
||||
"required": [
|
||||
"verdicts"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"$defs": {
|
||||
@@ -747,8 +813,198 @@
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"verdict": {
|
||||
"$ref": "#/$defs/verdict"
|
||||
},
|
||||
"verdict_evidence": {
|
||||
"$ref": "#/$defs/verdict_evidence"
|
||||
},
|
||||
"exit": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"status",
|
||||
"signal"
|
||||
],
|
||||
"properties": {
|
||||
"status": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"signal": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"timeout_s": {
|
||||
"type": "number",
|
||||
"minimum": 0
|
||||
}
|
||||
},
|
||||
"description": "How a worker that did not finish ended: its exit status or the signal that killed it, or the timeout after which the probe killed it."
|
||||
}
|
||||
}
|
||||
},
|
||||
"verdict": {
|
||||
"enum": [
|
||||
"boots",
|
||||
"bios_error",
|
||||
"crash",
|
||||
"hang",
|
||||
"unknown"
|
||||
],
|
||||
"description": "boots: a reference frame of the content was reached. bios_error: an error pattern of the core log or a known error screen, and no reference frame. crash: the worker died. hang: killed after the timeout, or every sampled frame of the second half is one colour. unknown: none of these."
|
||||
},
|
||||
"verdict_evidence": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"rule",
|
||||
"detail"
|
||||
],
|
||||
"properties": {
|
||||
"rule": {
|
||||
"enum": [
|
||||
"reference_frame",
|
||||
"error_pattern",
|
||||
"error_screen",
|
||||
"exit_status",
|
||||
"timeout",
|
||||
"uniform_tail",
|
||||
"no_evidence",
|
||||
"repetitions_agree",
|
||||
"repetitions_disagree"
|
||||
]
|
||||
},
|
||||
"detail": {
|
||||
"type": "string",
|
||||
"minLength": 1
|
||||
},
|
||||
"frame": {
|
||||
"type": "integer",
|
||||
"minimum": 0
|
||||
},
|
||||
"sha1": {
|
||||
"$ref": "#/$defs/sha1"
|
||||
},
|
||||
"stages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/stage"
|
||||
},
|
||||
"description": "Stages of the reference frames the run reached, in the order first reached."
|
||||
},
|
||||
"pattern": {
|
||||
"type": "string"
|
||||
},
|
||||
"log": {
|
||||
"type": "string"
|
||||
},
|
||||
"exit_status": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"signal": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"repetitions": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/verdict"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"stage": {
|
||||
"enum": [
|
||||
"bios",
|
||||
"game"
|
||||
],
|
||||
"description": "bios: a screen only the console's own BIOS draws (its logo). game: a screen of the content itself."
|
||||
},
|
||||
"oracle": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "null"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"file",
|
||||
"sha256",
|
||||
"references",
|
||||
"error_patterns",
|
||||
"error_screens"
|
||||
],
|
||||
"properties": {
|
||||
"file": {
|
||||
"type": "string"
|
||||
},
|
||||
"sha256": {
|
||||
"$ref": "#/$defs/sha256"
|
||||
},
|
||||
"references": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"sha1",
|
||||
"stage",
|
||||
"shows"
|
||||
],
|
||||
"properties": {
|
||||
"sha1": {
|
||||
"$ref": "#/$defs/sha1"
|
||||
},
|
||||
"stage": {
|
||||
"$ref": "#/$defs/stage"
|
||||
},
|
||||
"shows": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"error_patterns": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"error_screens": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"sha1",
|
||||
"shows"
|
||||
],
|
||||
"properties": {
|
||||
"sha1": {
|
||||
"$ref": "#/$defs/sha1"
|
||||
},
|
||||
"shows": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"description": "What the verdicts were judged against: the reference frames of this content and core, the core's error patterns and error screens, from the oracle file at its sha256. Null when the run had no oracle entry."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,8 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
RECORD_VERSION = 3
|
||||
# 3: before the boot oracle. 4: every repetition and variant carries a verdict.
|
||||
RECORD_VERSIONS = (3, 4)
|
||||
RECORDS_DIR = "measurements"
|
||||
ID_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$")
|
||||
SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:[-_][a-z0-9]+)*$")
|
||||
@@ -137,8 +139,8 @@ def check_line(record: dict) -> list[str]:
|
||||
"""The structural rules a line holds, without a JSON Schema validator."""
|
||||
rid = record.get("id", "")
|
||||
problems = []
|
||||
if record.get("record_version") != RECORD_VERSION:
|
||||
problems.append(f"{rid}: record_version {record.get('record_version')} != {RECORD_VERSION}")
|
||||
if record.get("record_version") not in RECORD_VERSIONS:
|
||||
problems.append(f"{rid}: record_version {record.get('record_version')} not in {RECORD_VERSIONS}")
|
||||
if not ID_PATTERN.match(str(rid)):
|
||||
problems.append(f"{rid!r}: not an id of the form YYYY-MM-DD-<12 hex>")
|
||||
elif str(record.get("recorded_at", ""))[:10] != rid[:10]:
|
||||
|
||||
@@ -43,16 +43,21 @@ def repetition(value: str, frames: int = 3) -> dict:
|
||||
"last_frame_size": [4, 2], "tracing": "strace", "input_queries": 3, "input_pressed": 1,
|
||||
"files_opened": [{"path": "card.bin", "sha1_after": SHA1, "sha1_staged": SHA1}],
|
||||
"files_probed_absent": [],
|
||||
"verdict": "boots",
|
||||
"verdict_evidence": {"rule": "reference_frame", "detail": "reached the title", "frame": 150,
|
||||
"sha1": value, "stages": ["game"]},
|
||||
}
|
||||
|
||||
|
||||
def full_record(emulator: str, subject: str, when: str, kind: str = "check",
|
||||
values: tuple[str, ...] = ("1" * 40, "2" * 40), observable: str = "video_sha1") -> dict:
|
||||
variants = [{"label": f"v{i}", "system_files": {"card.bin": SHA1},
|
||||
"repetitions": [repetition(v), repetition(v)], "flags": []}
|
||||
"repetitions": [repetition(v), repetition(v)], "flags": [], "verdict": "boots",
|
||||
"verdict_evidence": {"rule": "repetitions_agree", "detail": "both boot",
|
||||
"repetitions": ["boots", "boots"]}}
|
||||
for i, v in enumerate(values)]
|
||||
record = {
|
||||
"record_version": mr.RECORD_VERSION, "emulator": emulator, "subject": subject,
|
||||
"record_version": 4, "emulator": emulator, "subject": subject,
|
||||
"supersedes": None, "kind": kind, "recorded_at": when,
|
||||
"core": {"file": "fake_libretro.so", "sha256": SHA256, "repo": "https://example.org/fake",
|
||||
"commit": "c" * 40, "build_command": "make", "compiler": ["GCC 13"],
|
||||
@@ -67,7 +72,10 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check",
|
||||
"options": {"fake_option": "a"}, "env": {}, "frames": 3, "warmup": 0,
|
||||
"repetitions": 2 if kind == "check" else 5, "presses": [],
|
||||
"observables": ["frame_sha1", "video_sha1", "audio_sha1", "state_sha1", "frame_hashes"],
|
||||
"frame_hash_every": 150, "joypad_ports": 2},
|
||||
"frame_hash_every": 150, "joypad_ports": 2,
|
||||
"oracle": {"file": "tests/campaign/oracle.yml", "sha256": SHA256,
|
||||
"references": [{"sha1": values[0], "stage": "game", "shows": "title"}],
|
||||
"error_patterns": ["BIOS not found"], "error_screens": []}},
|
||||
"variants": variants,
|
||||
"flags": [],
|
||||
}
|
||||
@@ -81,7 +89,8 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check",
|
||||
"observable": observable, "variable": "card.bin",
|
||||
"result": {"status": "differs" if len(groups) > 1 else "same",
|
||||
"statement": f"{observable} takes {len(groups)} values",
|
||||
"groups": [{"value": json.loads(k), "variants": v} for k, v in groups.items()]},
|
||||
"groups": [{"value": json.loads(k), "variants": v} for k, v in groups.items()],
|
||||
"verdicts": {v["label"]: v["verdict"] for v in variants}},
|
||||
}
|
||||
else:
|
||||
record["variants"] = record["variants"][:1]
|
||||
@@ -304,6 +313,26 @@ class ReadAndIndex(Workspace):
|
||||
self.assertTrue(any("measurements/beta.jsonl:1" in e and "kind" in e for e in errors), errors)
|
||||
self.assertIsNotNone(Draft202012Validator)
|
||||
|
||||
def test_a_version_4_record_needs_its_verdicts(self) -> None:
|
||||
try:
|
||||
from jsonschema import Draft202012Validator # noqa: F401
|
||||
except ImportError:
|
||||
self.skipTest("jsonschema is not installed")
|
||||
import validate_schemas # noqa: PLC0415
|
||||
|
||||
lines = self.tree / mr.records_path("beta")
|
||||
record = json.loads(lines.read_text())
|
||||
del record["variants"][0]["verdict"]
|
||||
del record["variants"][0]["repetitions"][0]["verdict_evidence"]
|
||||
lines.write_text(mr.line_text(record) + "\n")
|
||||
errors = validate_schemas._measurement_errors(self.tree)
|
||||
self.assertTrue(any("'verdict' is a required property" in e for e in errors), errors)
|
||||
self.assertTrue(any("'verdict_evidence' is a required property" in e for e in errors), errors)
|
||||
old = {**record, "record_version": 3}
|
||||
lines.write_text(mr.line_text(old) + "\n")
|
||||
self.assertFalse(any("required property" in e
|
||||
for e in validate_schemas._measurement_errors(self.tree)))
|
||||
|
||||
|
||||
class AppendOnly(Workspace):
|
||||
def setUp(self) -> None:
|
||||
|
||||
Reference in new issue
Block a user