From 3f93da6394154f2974f4c29381a6d2e6f6e5529a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:52:29 +0100 Subject: [PATCH] docs: show boot verdicts on the measurement pages The Tested blocks and the explorer show each variant's verdict. A frame comparison from before the oracle shows no verdict. The SQLite variants table gains verdict columns. The method page says what each verdict means and how a file is proven. --- scripts/sitemeasurements.py | 64 +++++++++++++++++++++++------- tests/test_measurements.py | 21 +++++++++- wiki/measurements.md | 39 ++++++++++++------- wiki/tools.md | 78 +++++++++++++++++++++++++++++++------ 4 files changed, 161 insertions(+), 41 deletions(-) diff --git a/scripts/sitemeasurements.py b/scripts/sitemeasurements.py index ac420682f..cc35ef362 100644 --- a/scripts/sitemeasurements.py +++ b/scripts/sitemeasurements.py @@ -39,7 +39,7 @@ HOSTS_MARKER = "" INDEX_PATH = Path("data") / "measurements" / "index.json" SQLITE_PATH = Path("downloads") / "measurements.sqlite" PER_EMULATOR_DIR = Path("downloads") / "measurements" -SQLITE_SCHEMA_VERSION = "2" +SQLITE_SCHEMA_VERSION = "3" # Observables the variants table leaves out: one value per sampled frame. BULKY_OBSERVABLES = ("frame_hashes",) INDEX_SCHEMA_VERSION = 1 @@ -74,6 +74,40 @@ def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "") +def verdicts_of(record: dict) -> list[dict]: + """Each variant's boot verdict, from record version 4 on.""" + return [{"variant": v["label"], "verdict": v["verdict"], + "detail": (v.get("verdict_evidence") or {}).get("detail", "")} + for v in record["variants"] if "verdict" in v] + + +def result_of(record: dict) -> tuple[str, str]: + """The status and statement a page shows for a record. + + A run, or a check compared on the sampled frames, is shown by its boot + verdicts: one verdict when every variant has it, else `mixed`, each + variant named in the statement. A check on another observable keeps + its comparison. A frame comparison recorded before the boot oracle + shows `no verdict`: frames that differ prove nothing about a boot. + """ + check = record.get("check") or {} + result = check.get("result") or {} + if record["kind"] == "bench": + return "measured", bench_statement(record) if record.get("timing") else "" + verdicts = verdicts_of(record) + by_frames = record["kind"] == "run" or check.get("observable") == "frame_hashes" + if verdicts and by_frames: + values = {v["verdict"] for v in verdicts} + status = values.pop() if len(values) == 1 else "mixed" + if record["kind"] == "run": + return status, verdicts[0]["detail"] + return status, ". ".join(f"{v['variant']}: {v['verdict']}" for v in verdicts) + if by_frames: + return "no verdict", ("recorded before the boot oracle. Its frames were compared, which " + "settles nothing about a boot") + return result.get("status") or "ran", result.get("statement") or "" + + def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict, system_names: dict[str, str]) -> dict: record = entry.record @@ -93,9 +127,9 @@ def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict, "method": check.get("method"), "observable": check.get("observable"), "variable": check.get("variable"), - "status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"), - "statement": (check.get("result") or {}).get("statement") - or (bench_statement(record) if record.get("timing") else ""), + "status": result_of(record)[0], + "statement": result_of(record)[1], + "verdicts": verdicts_of(record), "content": content.get("label"), "variants": len(record["variants"]), "repetitions": record["inputs"]["repetitions"], @@ -292,7 +326,8 @@ SCHEMA_SQL = """ run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL, label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL, flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT, - files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position) + files_probed_absent TEXT, tracing TEXT, verdict TEXT, verdict_evidence TEXT, + PRIMARY KEY (run_id, position) ); CREATE TABLE system_files ( run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL, @@ -403,11 +438,12 @@ def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict, # The first repetition stands for the variant, its per-frame hashes left out. observables = {key: value for key, value in (first.get("observables") or {}).items() if key not in BULKY_OBSERVABLES} - connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( + connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)), json_text(variant["flags"]), json_text(observables), json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")), - json_text(first.get("tracing")))) + json_text(first.get("tracing")), variant.get("verdict"), + json_text(variant.get("verdict_evidence")) if "verdict_evidence" in variant else None)) connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [ (rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())]) for metric, stats in sorted((r.get("timing") or {}).items()): @@ -503,12 +539,14 @@ def tested_item(item: dict) -> list[str]: core = item["core"] build = f"{_text(core['library'])} at {_short(core['commit'])}" if core.get("commit") \ else _text(core["library"]) - lines = [ - f"- **{_text(title)}**", - "", - f" **Result:** `{item['status']}`. {_text(item['statement'])}.", - "", - ] + lines = [f"- **{_text(title)}**", ""] + if len(item.get("verdicts") or ()) > 1: + lines += [f" **Result:** `{item['status']}`.", ""] + lines += [f" - {_text(v['variant'])}: `{v['verdict']}`. {_text(v['detail'])}." + for v in item["verdicts"]] + lines.append("") + else: + lines += [f" **Result:** `{item['status']}`. {_text(item['statement'])}.", ""] if item.get("method"): lines += [f" **Method:** {_text(item['method'])}", ""] lines += [ diff --git a/tests/test_measurements.py b/tests/test_measurements.py index 4ea69407a..c7eb2536e 100644 --- a/tests/test_measurements.py +++ b/tests/test_measurements.py @@ -220,7 +220,7 @@ class ReadAndIndex(Workspace): keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind", "claim", "method", "observable", "variable", "status", "statement", "content", "variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id", - "runs", "record_url", "sqlite"} + "runs", "record_url", "sqlite", "verdicts"} for item in index["items"]: self.assertEqual(set(item), keys) alpha = {i["subject"]: i for i in index["items"] if i["emulator"] == "alpha"} @@ -234,6 +234,25 @@ class ReadAndIndex(Workspace): written = json.loads((self.tmp / "public" / sm.INDEX_PATH).read_text()) self.assertEqual(written, index) + def test_a_frame_check_shows_its_verdicts_never_frames_that_differ(self) -> None: + record = full_record("alpha", "boot", "2026-10-06T10:00:00+00:00", observable="frame_hashes") + record["variants"][1]["verdict"] = "hang" + self.assertEqual(sm.result_of(record), ("mixed", "v0: boots. v1: hang")) + self.assertEqual([v["verdict"] for v in sm.verdicts_of(record)], ["boots", "hang"]) + old = {**record, "record_version": 3, + "variants": [{k: v for k, v in variant.items() if not k.startswith("verdict")} + for variant in record["variants"]]} + status, statement = sm.result_of(old) + self.assertEqual(status, "no verdict") + self.assertNotIn("differ", statement) + # A check on another observable keeps its comparison. + self.assertEqual(sm.result_of(full_record("alpha", "c", "2026-10-06T10:00:00+00:00"))[0], "differs") + item = {"claim": "c", "runs": 1, "core": {"library": "Fake 1", "commit": None}, "status": "mixed", + "statement": "", "verdicts": sm.verdicts_of(record), "method": "m", "host": "h", + "date": "2026-10-06", "variants": 2, "repetitions": 2, "flags": 0, "record_url": "u"} + text = "\n".join(sm.tested_item(item)) + self.assertIn("v1: `hang`", text) + def test_links_name_the_last_commit_that_touched_the_records(self) -> None: first = self.repository() (self.tree / "other.txt").write_text("x") diff --git a/wiki/measurements.md b/wiki/measurements.md index 3543fb127..b251058d0 100644 --- a/wiki/measurements.md +++ b/wiki/measurements.md @@ -14,25 +14,34 @@ the emulator's page shows a **Tested** block. Every result is on the ## What a test does - A script runs the emulator with no window and no sound. -- It starts a game or a disc and presses buttons at set frames. -- It runs again for each version of one file, such as a BIOS. Nothing else changes. -- It compares what the emulator showed, played or saved, by fingerprint (a hash). -- Each version runs at least twice, each time from a fresh start. - -## When tests run - -A maintainer starts a test run. It then works alone on an idle computer. A clear -result updates the emulator's page and names the test as proof. - -Anything else waits on the [Measurements](../measurements.md) page for someone -to read the code. +- It starts a game and runs it for a set time. +- Each run starts fresh and runs at least twice. +- Each run gets one result, by comparing its screens with ones we checked by eye. ## What a result says -- **same**: the file changed nothing that was compared. -- **differs**: some versions of the file changed it. -- **inconclusive**: two runs of one version did not match, so no answer is given. +- **boots**: the game reached a screen it shows when it works. +- **bios_error**: the emulator said a BIOS is missing or wrong. +- **crash**: the emulator stopped. +- **hang**: the screen stayed blank, or the run never ended. +- **unknown**: none of these could be shown. - **measured**: a speed test, given as time per frame. +- **no verdict**: an older test that only compared screens. + +## How a file is proven + +The game runs with no BIOS, with the BIOS alone, and with a wrong file under +its name: zeros, half the file, or another region's BIOS. + +- **Required**: no BIOS fails, the BIOS alone boots. +- **Built-in stand-in**: no BIOS still boots. +- **Checked**: a wrong file is refused. + +## When tests run + +A maintainer starts a test run. A clear result updates the emulator's page and +names the test as proof. Anything else waits on the +[Measurements](../measurements.md) page for someone to read the code. ## What a result keeps diff --git a/wiki/tools.md b/wiki/tools.md index 19d3ee0ef..2fd18ab54 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -845,6 +845,12 @@ python scripts/core_probe.py bench --core core.so --content game.cue \ python scripts/core_probe.py check --core core.so --content disc.cue \ --vary mpeg_card.bin=a.bin --vary mpeg_card.bin=b.bin --vary mpeg_card.bin= \ --observable video_sha1 --claim "..." --method "..." --record + +# negative variants: the file, zeros of its size, its first half +python scripts/core_probe.py check --core core.so --content disc.cue \ + --frame-hash-every 10 --vary scph5501.bin=bios/scph5501.bin \ + --vary scph5501.bin=@zeros:524288 --vary scph5501.bin=@head:262144:bios/scph5501.bin \ + --observable frame_hashes --claim "..." --method "..." --repeat 2 --record ``` Each repetition runs in a fresh process under `strace -f -e trace=openat,open` @@ -859,15 +865,41 @@ writable. Every namespace is unshared, the network included, so nothing of the user (`~/.config`, `~/.ssh`, `~/.git-credentials`) or of the checkout is there to read. Each repetition names what it ran under in `isolation` (`bwrap`, or `environment only` with the reason), and `--require-sandbox` -refuses to run without it. A timeout kills the worker's whole process group. The files the core opened under the system +refuses to run without it. The files the core opened under the system directory come from the kernel, each hashed again after the run. Without strace the record says that tracing was unavailable. +**Boot oracle.** Every repetition and every variant carries a `verdict` and +the `verdict_evidence` that decided it (record version 4), judged against +`tests/campaign/oracle.yml` (`--oracle`), in this order: + +| Verdict | Evidence | +|---------|----------| +| `hang` | the worker was killed after `--timeout` (`timeout`) | +| `crash` | the worker died, by exit status or signal (`exit_status`). Its logs and sampled frames are rebuilt from the events it streamed | +| `boots` | a sampled frame or the last frame is a `game` reference frame of this content on this core (`reference_frame`), with the `stages` reached: `bios` for a screen only the console's BIOS draws, `game` for one of the content. The BIOS screens alone are never a boot: the stages go into the evidence of the verdict below | +| `bios_error` | a log line matches an error pattern of the core (`error_pattern`), or a frame is one of its error screens (`error_screen`) | +| `hang` | every sampled frame of the second half is one colour (`uniform_tail`) | +| `unknown` | none of these (`no_evidence`) | + +A variant's verdict is the one all its repetitions agree on, else `unknown` +(`repetitions_disagree`). A check lists them in `check.result.verdicts`, and +`inputs.oracle` keeps what they were judged against. The oracle finds a +content by the sha1 of the file handed to the core and a core by its library +name. A reference frame is recorded once with the full BIOS set and reviewed +by looking at its PNG. An error pattern is a log line after which the core +stops, read from its source: a line a core logs before it falls back to a +built-in BIOS is not one. Frames that differ from another run never make a +boot. + | Option | Effect | |--------|--------| | `--system-file DEST=SRC` | copy SRC into a staged system directory as DEST | | `--content-label` | identity of the content, such as its Redump name, copied into the record | -| `--work-dir`, `--timeout` | scratch directory for the staged runs, and seconds allowed per repetition (default 1800) | +| `--work-dir`, `--timeout` | scratch directory for the staged runs, and seconds allowed per repetition (default 1800), after which the run is killed and judged a hang | +| `--oracle FILE` | reference frames, error patterns and error screens the verdicts are judged against (default `tests/campaign/oracle.yml`) | +| `--repeat N` | runs of each variant, each in a fresh process (`run` defaults to 1, `check` to 2, `bench` to 5) | +| `--vary DEST=SRC` | (`check`) one variant per occurrence. An empty SRC leaves DEST absent, `@zeros:SIZE` stages SIZE zero bytes and `@head:SIZE:PATH` the first SIZE bytes of PATH | | `--option KEY=VALUE` | core option. Unset options take the first value the core declares | | `--frames`, `--warmup` | frames timed, and frames run before timing starts | | `--press FRAME:BUTTON[:LEN]` | hold a libretro joypad button on port 0 (`a`, `b`, `start`, `up`...). A menu may need 15 frames or more to see it | @@ -947,6 +979,8 @@ files a core opens nor writes a record. python scripts/campaign.py run --system sony-playstation --budget 6h python scripts/campaign.py run --only beetle_psx,swanstation --dry-run python scripts/campaign.py run --commit --commit-every 10 +python scripts/campaign.py run --only pcsx_rearmed --bench # with benchmarks +python scripts/campaign.py references --only beetle_psx --system sony-playstation python scripts/campaign.py status # done, failed, pending, and why python scripts/campaign.py findings # what waits for review python scripts/campaign.py gate # the checks behind a commit @@ -957,18 +991,37 @@ python scripts/campaign.py install-timer # opt-in: writes the autorun units, |------|--------------| | Core | The x86_64 Linux nightly from `buildbot.libretro.com`, for each name of the profile's `cores:` the buildbot publishes, downloaded again only when its `ETag` or `Last-Modified` changes. The record keeps the artifact URL, the `.so` sha256 and the `retro_get_system_info` version. With no nightly, the profile's `source_commit` is cloned and built once with `make` (`Makefile.libretro` first), the failure kept so it is not tried again for that commit. Host git only fetches into bare mirrors (`.cache/campaign/mirrors/`), one per repository, submodules included, that it alone touches: no working tree, never bound into a sandbox, and earlier working-tree clones are deleted. It runs with no user or system config, https as the only transport, and `-c` overrides emptying hooks, `core.fsmonitor`, `core.sshCommand`, `core.gitProxy`, credential helpers and every filter driver the mirror's config defines. The commit and each submodule at the commit its tree records are exported with `git archive` into a fresh folder, and `make` runs there in the same bubblewrap sandbox as a core, that folder the one writable path. The core copied out is the newest `*_libretro.so` that is a regular file inside that folder, opened without following links. The record's `build_command` says which isolation was used. A standalone emulator is `pending: standalone` | | Content | Each system's games come from `tests/campaign/content.yml`, by catalog name and with the reason each was chosen. They are read from `.cache/test-content///`, each file checked against its `identity.json`. Missing content is fetched with `romfetch get --json` when romfetch is on `PATH`, in `RETROBIOS_ROMFETCH` or given with `--romfetch`. Otherwise the checks are `pending: content`, with romfetch's exit code and error code when it failed. Content never enters git, and a record keeps only its identity | -| Checks | `files` stages every declared file the collection holds and reads which ones the core opens (strace). `boot` runs the game with no file and with the declared one (`boot_file:`, else the entry of the game's region). An entry the `files` run left unread is staged `alone`, then where the core looked for it (`path`) and under the names it looked for (`alias`). `variants` runs each `.variants/` copy of a file the core opened. `bench` times the game, warm-up excluded, 5 repetitions | +| Checks | `files` stages every declared file the collection holds (the full set) and reads which ones the core opens (strace). `none` runs the game with no BIOS at all. `negative` runs, for each declared file of the game's region the collection holds, the file alone, then zeros of its size, its first half and up to two other regions' dumps staged under its name. An entry neither the full set nor its own run opened is staged where the core looked for it (`path`), else flagged. The `boot_file:` entry is also staged under the names the core looked for with no BIOS (`alias`). `variants` runs each `.variants/` copy of a file the core opened. Each runs twice from a fresh process and is judged by the boot oracle. `bench` (only with `--bench`) times the game once the full set boots, warm-up excluded, 5 repetitions | | Rules | `campaign_rules.py`, a fixed table of pure functions (below). A field the table settles is written into the profile, with `evidence:` naming the record ids. Anything else is a line of `measurements/findings.jsonl` | | Content reuse | A game stays on disk until the last emulator that needs it in the pass is done, then the campaign deletes it if it fetched it. Past `--cache-cap-gb` (default 20) the least recently used goes first. An entry marked `pinned:`, or one the campaign did not fetch, is never deleted | -| Observable | Field | Written | -|------------|-------|---------| -| Without the file, at least 2 screens the run with the file also shows (single-colour frames left out), in every repetition | `hle_fallback` | `true` | -| Without the file, no repetition loads the game or shows more than one colour, while the run with it reaches the game | `required`, `hle_fallback` | `true`, `false` | +| Verdicts | Field | Written | +|----------|-------|---------| +| `none` boots | `hle_fallback` | `true` | +| `none` fails (`bios_error`, `crash`, `hang`) and the file alone boots | `required`, `hle_fallback` | `true`, `false` | +| Zeros of the file's size refused while the file boots | `validation` | a content check is proven. Agreeing adds evidence, a profile with no md5, sha1, sha256, crc32 or adler32 is an open finding | +| Zeros used while the file boots | `validation` | no content check. A profile that declares one is an open finding | +| Its first half refused while zeros are used | `validation` | `size`, added when missing | +| Its first half used | `validation` | no size check. A profile that declares one is an open finding | | The core opened the file staged at another place it looked for | `path` | that path | | The core opened the file staged under a name no entry declares | `aliases` | the name added | | A staged file the game's region uses, never opened, staged alone either | nothing | flagged | -| Anything else (runs that disagree, a profile that says `required: true` for a file the core boots without) | nothing | open finding | +| Copies of one file whose verdicts differ | nothing | open finding | +| Anything else (an `unknown`, a profile that says `required: true` for a file the core boots without) | nothing | open finding | + +A wrong file is refused when the run reports `bios_error`, or boots without +the `bios` screens the right file shows: the core fell back to a built-in +BIOS. It is used when it turns the run into a `hang` or a `crash` that the +run with no BIOS does not show. Another region's dump is recorded with its +verdict, and the region claims are settled by their own test. Evidence that a +record from before the oracle wrote no longer counts: when a verdict settles a +field again, those ids leave its `evidence:`. + +`references` runs the full set once per core and content with every sampled +frame written as PNG under `tmp/campaign/references///`, and +prints the record id with the screens that stay on for 3 samples or more in +every repetition. A reviewer opens those PNGs and writes the chosen frames +into `tests/campaign/oracle.yml` with their stage and what they show. A run that agrees with the profile adds its record id to `evidence:` and resolves an open finding of the same field. The profile is edited as text and @@ -977,8 +1030,9 @@ those fields changed. State lives in `.cache/campaign/state.json` of the main checkout: each check is `done`, `failed` (with the reason and its log) or `pending` (with the -reason), with the core sha256 and the profile digest it ran against. A pass -skips a check that is done or failed against the same core and profile, so it +reason), with the core sha256, the profile digest, the oracle sha256 and the +record version it ran against. A pass skips a check that is done or failed +against the same four, so it takes new nightlies, changed profiles, newly fetched content and pending checks. The campaign's own `evidence:` writes do not count as a profile change. @@ -990,8 +1044,8 @@ checks. The campaign's own `evidence:` writes do not count as a profile change. | Load | Before each run, waits while the 1-minute load is above a quarter of the CPUs, up to `--max-load-wait` seconds, then the pass stops as `paused` | | Disk | Below `--min-free-gb` (default 30) the pass stops as `paused` | | Budget | No check starts after `--budget` | -| Timeout | A run is killed after 900 seconds per repetition and variant, recorded `failed: timeout`, and the pass moves on | -| Crash | A run that dies is tried 3 times, then `failed` with its log | +| Timeout | A repetition is killed after 900 seconds and judged `hang`. A core_probe call that outlives all its repetitions is `failed: timeout` | +| Crash | A core that dies is a `crash` verdict in its record. A core_probe call that fails for another reason is tried 3 times, then `failed` with its log | | Network | Buildbot and romfetch source errors are retried after 10, 60 and 300 seconds | Each pass appends one line to `measurements/campaign-runs.jsonl`: when it ran,