chore: merge region and coverage checks

scripts/campaign.py

@@ -1,41 +1,56 @@
 #!/usr/bin/env python3
-"""Run the real-condition campaign unattended: cores, content, checks, rules.
+"""Run the real-condition campaign end to end: cores, content, checks, rules.
 
-For each emulator profile in scope, the campaign takes the core (the libretro
-x86_64 Linux nightly, else a build of the profile's pinned commit), the test
-content of each of its systems (tests/campaign/content.yml, fetched through
-romfetch when it is not on disk), and runs core_probe.py:
+A maintainer starts each pass. For each emulator profile in scope, the
+campaign takes the core (the libretro x86_64 Linux nightly, else a build of
+the profile's pinned commit), the test content of each of its systems
+(tests/campaign/content.yml, fetched through romfetch when it is not on
+disk), and runs core_probe.py. The boot oracle of core_probe judges every
+run: boots, bios_error, crash, hang or unknown, against the reference frames
+and error patterns of tests/campaign/oracle.yml.
 
-  files      every declared file the collection holds, staged, and which ones
-             the core opens
-  boot       the content with no file against the content with the declared
-             file, compared on the frames the core shows
+  files      every declared file the collection holds, staged: the full set,
+             and which files the core opens
+  none       no BIOS at all
+  negative   per declared file of the content's region: the file alone, then
+             zeros of its size, its first half and other regions' dumps
+             staged under its name
   path       a declared file staged where the core looked for it
   alias      a declared file staged under a name the core looked for
   variants   each .variants/ copy of a file the core opened
-  bench      the content timed, warm-up excluded, 5 repetitions
+  bench      with --bench only: the content timed, warm-up excluded
 
 Every run is a record in measurements/<emulator>.jsonl. The fixed rule table
-(campaign_rules.py) turns records into verdicts. A verdict the table settles
-is written into the profile with the record ids as `evidence:`. Any other
-goes to measurements/findings.jsonl, where the site lists it as open. Each
-pass adds one line to measurements/campaign-runs.jsonl.
+(campaign_rules.py) reads the verdicts, never a comparison of frames. A
+verdict the table settles is written into the profile with the record ids
+as `evidence:`. Any other goes to measurements/findings.jsonl, where the
+site lists it as open. Each pass adds one line to
+measurements/campaign-runs.jsonl.
+
+`references` runs the full set once per core and content with its frames
+saved as PNG, and lists the screens that stay on for several samples: the
+candidates a reviewer looks at before writing them into oracle.yml.
 
 State is .cache/campaign/state.json of the main checkout: per check done,
-failed or pending with the reason, and the core sha256 and profile digest it
-ran against. A pass skips what is done unless the core build or the profile
-changed. The campaign holds the host-wide automation lock, waits for an idle
-host, stops cleanly on a busy host or a full disk, and resumes on the next
-pass. Under autorun (`campaign` job) the lock is the job's.
+failed or pending with the reason, and the core sha256, profile digest,
+oracle sha256 and record version it ran against. A pass skips what is done
+unless one of them changed. The campaign holds the host-wide automation
+lock, waits for an idle host, stops cleanly on a busy host or a full disk,
+and the next pass a maintainer starts resumes there.
 
 Usage:
-    python scripts/campaign.py run [--only CORE,...] [--system ID] [--budget 6h]
+    python scripts/campaign.py run [--only CORE,...] [--system ID] [--budget 6h] [--bench]
     python scripts/campaign.py run --dry-run
     python scripts/campaign.py run --commit [--commit-every N]
+    python scripts/campaign.py references --only CORE,... [--system ID]
     python scripts/campaign.py status [--json]
     python scripts/campaign.py findings [--all] [--json]
     python scripts/campaign.py gate
     python scripts/campaign.py install-timer [--enable]
+
+install-timer is opt-in: a contributor who wants a schedule on their own
+machine installs the timer. It writes the units and enables nothing unless
+--enable is passed.
 """
 
 from __future__ import annotations
@@ -86,6 +101,7 @@ PENDING = "pending"
 CRASH_ATTEMPTS = 3
 NETWORK_DELAYS = (10, 60, 300)
 MAX_ALIAS_CANDIDATES = 6
+MAX_REGION_DUMPS = 2
 BENCH_WARMUP = 600
 BENCH_FRAMES = 1800
 BENCH_REPEAT = 5
@@ -175,6 +191,11 @@ def region_matches(entry: dict, region: str | None) -> bool:
     return region in regions or "world" in regions
 
 
+def regions_of(entry: dict) -> list[str]:
+    regions = entry.get("region") or []
+    return [regions] if isinstance(regions, str) else list(regions)
+
+
 def is_bios(entry: dict) -> bool:
     return entry.get("category", "bios") == "bios" and entry.get("type") != "directory"
 
@@ -235,6 +256,8 @@ class Settings:
     bench_frames: int = BENCH_FRAMES
     romfetch: str = ""
     network: bool = True
+    # Benchmarks only choose between variants: never part of a default pass.
+    bench: bool = False
     delays: tuple[float, ...] = NETWORK_DELAYS
     echo: bool = True
     # Third-party code (a nightly core, a project's Makefile) runs only under
@@ -252,6 +275,7 @@ class Paths:
     content: Path
     db: Path
     content_file: Path = CONTENT_FILE
+    oracle: Path = core_probe.DEFAULT_ORACLE
 
     @property
     def state(self) -> Path:
@@ -279,9 +303,9 @@ def child_env(romfetch: bool = False) -> dict[str, str]:
     """What a program the campaign starts inherits: autorun's allowlist,
     never a token or a password. romfetch also gets its own settings
     (ROMFETCH_*), its account staying in its config file."""
-    names: list[str] = []
+    names: list[str] = [core_probe.PROBE_LOCK_ENV, core_probe.PROBE_LOCK_HELD_ENV]
     if romfetch:
-        names = [k for k in os.environ if k.startswith("ROMFETCH_")] + list(ROMFETCH_PASSED)
+        names += [k for k in os.environ if k.startswith("ROMFETCH_")] + list(ROMFETCH_PASSED)
     return autorun.safe_env(names)
 
 
@@ -470,14 +494,17 @@ def save_state(path: Path, state: dict) -> None:
     autorun.save_state(path, state)
 
 
-def should_run(previous: dict | None, core_sha: str, digest: str) -> bool:
+def should_run(previous: dict | None, core_sha: str, digest: str, oracle_sha: str = "") -> bool:
     """A check runs again when it never ran, is pending, or ran against
-    another core build or another version of the profile."""
+    another core build, another version of the profile, another oracle, or
+    before records carried a verdict."""
     if not previous:
         return True
     if previous.get("status") == PENDING:
         return True
-    return previous.get("core_sha256") != core_sha or previous.get("profile_digest") != digest
+    return (previous.get("core_sha256") != core_sha or previous.get("profile_digest") != digest
+            or previous.get("record_version") != mr.RECORD_VERSION
+            or previous.get("oracle_sha256", "") != oracle_sha)
 
 
 # --------------------------------------------------------------------------
@@ -875,6 +902,8 @@ class Campaign:
         self.cores = Cores(self)
         self.store = ContentStore(self)
         self.content = load_content(paths.content_file)
+        self.oracle_sha = file_sha256(paths.oracle) if paths.oracle.is_file() else ""
+        self.oracle_data = core_probe.load_oracle(paths.oracle)[0]
         self.deadline = 0.0
         self.current: Pass | None = None
         self.dirty_at_start: set[str] = set()
@@ -1004,6 +1033,8 @@ class Campaign:
                 "--emulator", emulator, "--subject", subject,
                 "--max-load", str(self.host.max_load), "--timeout", str(self.settings.probe_timeout),
                 "--work-dir", str(self.paths.root / "tmp" / "core_probe")]
+        if self.paths.oracle.is_file():
+            argv += ["--oracle", str(self.paths.oracle)]
         if not self.settings.allow_unsandboxed:
             argv.append("--require-sandbox")
         if core.artifact:
@@ -1025,8 +1056,14 @@ class Campaign:
 
     def remember(self, emulator: str, key: str, result: ProbeResult, core: Core | None,
                  digest: str) -> None:
+        judged = None
+        if result.record and result.record["inputs"].get("content"):
+            judged = [result.record["inputs"]["content"]["sha1"], result.record["core"]["library_name"]]
         entry = {"status": result.status, "at": now_iso(), "profile_digest": digest,
-                 "core_sha256": core.sha256 if core else None}
+                 "core_sha256": core.sha256 if core else None,
+                 "record_version": mr.RECORD_VERSION, "oracle_sha256": self.oracle_digest(judged)}
+        if judged:
+            entry["oracle_key"] = judged
         if result.reason:
             entry["reason"] = result.reason
         if result.log:
@@ -1042,6 +1079,38 @@ class Campaign:
         self.state["checks"].setdefault(emulator, {})[key] = {
             "status": PENDING, "reason": reason, "at": now_iso(), "profile_digest": digest}
 
+    def oracle_digest(self, key: list[str] | None) -> str:
+        """What a check's verdicts depend on in the oracle: the entries of its
+        content and core, else the whole file. Another core's references
+        never make a check run again."""
+        if not key:
+            return self.oracle_sha
+        return mr.value_sha1(core_probe.oracle_for(self.oracle_data, *key))
+
+    def needed(self, emulator: str, key: str, core: Core, digest: str) -> bool:
+        previous = self.state["checks"].get(emulator, {}).get(key)
+        judged = (previous or {}).get("oracle_key")
+        if previous and not judged and previous.get("record"):
+            record = self.read_record(emulator, previous["record"]) or {}
+            if (record.get("inputs") or {}).get("content"):
+                judged = [record["inputs"]["content"]["sha1"], record["core"]["library_name"]]
+                previous["oracle_key"] = judged
+                # Judged against the entries the record kept, not the file it named.
+                kept = {k: v for k, v in (record["inputs"].get("oracle") or {}).items()
+                        if k not in ("file", "sha256")}
+                previous["oracle_sha256"] = mr.value_sha1({"bios_patterns": [], **kept})
+        return should_run(previous, core.sha256, digest, self.oracle_digest(judged))
+
+    def check(self, emulator: str, key: str, core: Core, digest: str, argv: list[str],
+              log_name: str) -> dict | None:
+        """The record of a check: run now when needed, else the one kept."""
+        if not self.needed(emulator, key, core, digest):
+            previous = self.state["checks"].get(emulator, {}).get(key) or {}
+            return self.read_record(emulator, previous.get("record"))
+        result = self.probe(argv, log_name)
+        self.remember(emulator, key, result, core, digest)
+        return result.record
+
     def system_entries(self, profile: dict, system: str) -> list[tuple[int, dict]]:
         from validation import filter_files_by_mode
 
@@ -1060,84 +1129,148 @@ class Campaign:
         held.sort(key=lambda item: (item[1].get("priority", 1 << 30), item[0]))
         return held[0] if held else None
 
+    @staticmethod
+    def other_regions(entry: dict, staged: dict[str, tuple[int, dict, Path]], local: Path) -> list[Path]:
+        """Held dumps of the same system for regions this entry does not
+        serve, one per distinct file: the wrong-region variants."""
+        mine = set(regions_of(entry))
+        if not mine or "world" in mine:
+            return []
+        out: dict[str, Path] = {}
+        for _, other, path in staged.values():
+            theirs = set(regions_of(other))
+            if theirs and "world" not in theirs and not theirs & mine and path.resolve() != local.resolve():
+                out.setdefault(file_sha1(path), path)
+        return sorted(out.values(), key=lambda p: p.name)[:MAX_REGION_DUMPS]
+
     def run_unit_content(self, unit: Unit, core: Core, content: Content, digest: str) -> None:
         emulator, profile = unit.emulator, unit.profile
-        checks = self.state["checks"].setdefault(emulator, {})
         entries = self.system_entries(profile, content.system)
-        boot = self.boot_entry(entries, content)
-        keys = {name: self.check_key(core, content, name) for name in ("files", "bench")}
-        if boot:
-            keys["boot"] = self.check_key(core, content, f"boot:{boot[1]['name']}")
-        todo = [k for k in keys.values() if should_run(checks.get(k), core.sha256, digest)]
-        if not todo:
-            return
-        try:
-            folder = self.store.get(content)
-            load = load_file(folder, content.load)
-        except Unavailable as exc:
-            for key in todo:
-                self.pending(emulator, key, str(exc), digest)
-            return
-        frames = ["--frames", str(content.frames), "--frame-hash-every", str(content.frame_hash_every)]
-        stem = f"{core.name}-{slug(content.name)}"
-
-        # files: every declared file the collection holds, at its place.
         staged: dict[str, tuple[int, dict, Path]] = {}
         for index, entry in entries:
             local = self.local_file(entry)
             if local and entry_dest(entry) not in staged:
                 staged[entry_dest(entry)] = (index, entry, local)
-        files_record = None
-        if keys["files"] in todo:
-            argv = self.probe_args("run", core, emulator, slug(f"files-{stem}"), content, load) + frames
-            for dest, (_, _, local) in sorted(staged.items()):
-                argv += ["--system-file", f"{dest}={local}"]
-            result = self.probe(argv, f"{emulator}-{stem}-files")
-            self.remember(emulator, keys["files"], result, core, digest)
-            files_record = result.record
-        elif checks.get(keys["files"], {}).get("status") == DONE:
-            files_record = self.read_record(emulator, checks[keys["files"]].get("record"))
+        tested = {dest: item for dest, item in staged.items() if region_matches(item[1], content.region)}
+        keys = {"files": self.check_key(core, content, "files"),
+                "none": self.check_key(core, content, "none")}
+        for dest in tested:
+            keys[f"negative:{dest}"] = self.check_key(core, content, f"negative:{dest}")
+        if self.settings.bench:
+            keys["bench"] = self.check_key(core, content, "bench")
+        # The rules run on every pass, from the records kept when nothing
+        # needs to run again: a change of the table applies without a run.
+        try:
+            folder = self.store.get(content)
+            load = load_file(folder, content.load)
+        except Unavailable as exc:
+            for key in keys.values():
+                if self.needed(emulator, key, core, digest):
+                    self.pending(emulator, key, str(exc), digest)
+            return
+        frames = ["--frames", str(content.frames), "--frame-hash-every", str(content.frame_hash_every)]
+        repeat = ["--repeat", str(self.settings.check_repeat)]
+        stem = f"{core.name}-{slug(content.name)}"
 
-        boot_record = None
-        if boot and keys["boot"] in todo:
-            index, entry = boot
-            local = self.local_file(entry)
-            dest = entry_dest(entry)
-            argv = self.probe_args("check", core, emulator, slug(f"boot-{entry['name']}-{stem}"),
-                                   content, load) + frames
-            argv += ["--vary", f"{dest}=", "--vary", f"{dest}={local}", "--observable", "frame_hashes",
-                     "--repeat", str(self.settings.check_repeat),
-                     "--claim", f"{core.name} reaches the game on {content.name} without {entry['name']}",
-                     "--method", (f"Two variants, {entry['name']} absent and {entry['name']} as the "
-                                  f"collection holds it, each run {self.settings.check_repeat} times "
-                                  f"from a fresh process for {content.frames} frames with no button "
-                                  f"pressed. Every {content.frame_hash_every}th frame is hashed. The "
-                                  f"runs without the file are compared with the runs with it on the "
-                                  f"frames that are not one colour.")]
-            result = self.probe(argv, f"{emulator}-{stem}-boot")
-            self.remember(emulator, keys["boot"], result, core, digest)
-            boot_record = result.record
-            if boot_record:
-                absent, present = (v["label"] for v in boot_record["variants"])
-                verdicts = rules.rule_bios_need(rules.boot_outcome(boot_record, absent, present),
-                                                entry, boot_record["id"])
+        # files: every declared file the collection holds, at its place.
+        argv = self.probe_args("run", core, emulator, slug(f"files-{stem}"), content, load) + frames + repeat
+        for dest, (_, _, local) in sorted(staged.items()):
+            argv += ["--system-file", f"{dest}={local}"]
+        files_record = self.check(emulator, keys["files"], core, digest, argv, f"{emulator}-{stem}-files")
+        # none: no BIOS at all.
+        argv = self.probe_args("run", core, emulator, slug(f"none-{stem}"), content, load) + frames + repeat
+        none_record = self.check(emulator, keys["none"], core, digest, argv, f"{emulator}-{stem}-none")
+        if not files_record or not none_record:
+            return
+        full = rules.run_of(files_record, files_record["variants"][0]["label"])
+        none = rules.run_of(none_record, none_record["variants"][0]["label"])
+        self.log(f"{emulator}: {core.name} on {content.name}: full set {full.verdict}, "
+                 f"no BIOS {none.verdict}")
+
+        for dest, (index, entry, local) in sorted(tested.items()):
+            record = self.negative(unit, core, content, load, frames, digest, keys[f"negative:{dest}"],
+                                   dest, entry, local, staged)
+            if not record:
+                continue
+            labels = [v["label"] for v in record["variants"]]
+            present = rules.run_of(record, labels[0])
+            zeros, half = (rules.run_of(record, label) for label in labels[1:3])
+            for label in labels[3:]:
+                self.log(f"{emulator}: {rules.rule_region(rules.run_of(record, label), present, entry, content.region)}")
+            if dest in self.opened(files_record, 0) or dest in self.opened(record, 0):
+                verdicts = rules.rule_need(none, present, entry, rules.read(none, present, zeros, half))
+                validation = rules.rule_validation(zeros, half, none, present, entry)
+                if validation:
+                    verdicts.append(validation)
                 self.settle(emulator, content, core, index, entry, verdicts)
-                self.follow_aliases(unit, core, content, load, frames, digest, index, entry,
-                                    local, boot_record, absent)
+                variants = self.variant_files(entry, local)
+                if variants:
+                    self.run_variants(unit, core, content, load, frames, digest, index, entry, dest,
+                                      local, variants)
+            else:
+                self.moved(unit, core, content, load, frames, digest, index, entry, dest, local,
+                           files_record, record)
 
-        if files_record and keys["files"] in todo:
-            self.read_files(unit, core, content, load, frames, digest, staged, files_record)
+        boot = self.boot_entry(entries, content)
+        if boot and entry_dest(boot[1]) in tested:
+            index, entry = boot
+            self.follow_aliases(unit, core, content, load, frames, digest, index, entry,
+                                tested[entry_dest(entry)][2], none_record)
 
-        if keys["bench"] in todo:
+        if self.settings.bench and full.verdict == rules.BOOTS:
             argv = self.probe_args("bench", core, emulator, slug(f"bench-{stem}"), content, load)
             argv += ["--warmup", str(self.settings.bench_warmup), "--frames",
                      str(self.settings.bench_frames), "--repeat", str(self.settings.bench_repeat),
                      "--no-trace"]
             if boot:
-                local = self.local_file(boot[1])
-                argv += ["--system-file", f"{entry_dest(boot[1])}={local}"]
-            result = self.probe(argv, f"{emulator}-{stem}-bench")
-            self.remember(emulator, keys["bench"], result, core, digest)
+                argv += ["--system-file", f"{entry_dest(boot[1])}={self.local_file(boot[1])}"]
+            self.check(emulator, keys["bench"], core, digest, argv, f"{emulator}-{stem}-bench")
+
+    def negative(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
+                 digest: str, key: str, dest: str, entry: dict, local: Path,
+                 staged: dict[str, tuple[int, dict, Path]]) -> dict | None:
+        """The file alone, then wrong files under its name: zeros of its
+        size, its first half, and other regions' dumps."""
+        stem = f"{core.name}-{slug(content.name)}"
+        size = local.stat().st_size
+        others = self.other_regions(entry, staged, local)
+        argv = self.probe_args("check", core, unit.emulator, slug(f"negative-{entry['name']}-{stem}"),
+                               content, load) + frames
+        argv += ["--vary", f"{dest}={local}", "--vary", f"{dest}=@zeros:{size}",
+                 "--vary", f"{dest}=@head:{size // 2}:{local}"]
+        for other in others:
+            argv += ["--vary", f"{dest}={other}"]
+        regions = f", then {', '.join(p.name for p in others)} staged under its name" if others else ""
+        argv += ["--observable", "frame_hashes", "--repeat", str(self.settings.check_repeat),
+                 "--claim", (f"{core.name} boots {content.name} with {entry['name']} alone and "
+                             f"refuses a wrong file under its name"),
+                 "--method", (f"{entry['name']} alone as {dest}, then {size} zero bytes and its "
+                              f"first {size // 2} bytes{regions}, nothing else staged. Each run "
+                              f"{self.settings.check_repeat} times from a fresh process for "
+                              f"{content.frames} frames with no button pressed, every "
+                              f"{content.frame_hash_every}th frame hashed. The boot oracle judges "
+                              f"each variant against the reference frames of {content.name}.")]
+        return self.check(unit.emulator, key, core, digest, argv, f"{unit.emulator}-{stem}-negative")
+
+    def moved(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
+              digest: str, index: int, entry: dict, dest: str, local: Path, files_record: dict,
+              alone: dict) -> None:
+        """A file the core opened neither in the full set nor alone: where
+        else it looked for a file of that name, else flagged."""
+        probed = [p for record in (files_record, alone) for r in record["variants"][0]["repetitions"]
+                  for p in r.get("files_probed_absent") or ()]
+        others = [p for p in dict.fromkeys(probed) if p != dest
+                  and p.rsplit("/", 1)[-1].casefold() == dest.rsplit("/", 1)[-1].casefold()]
+        for candidate in others[:MAX_ALIAS_CANDIDATES]:
+            found = self.follow_up(unit, core, content, load, frames, digest, entry, local, candidate,
+                                   f"path:{entry['name']}:{candidate}")
+            if found and candidate in self.opened(found, 0):
+                verdict = rules.rule_path(entry, dest, rules.Opened(candidate, found["id"]))
+                if verdict:
+                    self.settle(unit.emulator, content, core, index, entry, [verdict])
+                    return
+        verdict = rules.rule_never_opened(entry, [files_record["id"], alone["id"]], content.name)
+        self.settle(unit.emulator, content, core, index, entry, [verdict])
 
     def read_record(self, emulator: str, rid: str | None) -> dict | None:
         if not rid:
@@ -1150,49 +1283,10 @@ class Campaign:
                 return json.loads(line)
         return None
 
-    def read_files(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
-                   digest: str, staged: dict[str, tuple[int, dict, Path]], record: dict) -> None:
-        """Which staged files the core opened: path follow-ups, variants, flags."""
-        emulator = unit.emulator
-        rep = record["variants"][0]["repetitions"][0]
-        if rep.get("files_opened") is None:
-            return
-        opened = {item["path"] for item in rep.get("files_opened") or ()}
-        probed = list(rep.get("files_probed_absent") or ())
-        for dest, (index, entry, local) in sorted(staged.items()):
-            if dest in opened:
-                variants = self.variant_files(entry, local)
-                if variants:
-                    self.run_variants(unit, core, content, load, frames, digest, index, entry, dest,
-                                      local, variants)
-                continue
-            if not region_matches(entry, content.region):
-                continue
-            # A fallback is read only when the files before it are absent:
-            # staged alone, it shows whether the core reads it at all.
-            alone = self.follow_up(unit, core, content, load, frames, digest, entry, local, dest,
-                                   f"alone:{entry['name']}")
-            if alone is None:
-                continue
-            if dest in self.opened(alone, 1):
-                continue
-            probed_alone = [p for r in alone["variants"][0]["repetitions"]
-                            for p in r.get("files_probed_absent") or ()]
-            others = [p for p in dict.fromkeys(probed + probed_alone) if p != dest
-                      and p.rsplit("/", 1)[-1].casefold() == dest.rsplit("/", 1)[-1].casefold()]
-            moved = False
-            for candidate in others[:MAX_ALIAS_CANDIDATES]:
-                record_c = self.follow_up(unit, core, content, load, frames, digest, entry, local,
-                                          candidate, f"path:{entry['name']}:{candidate}")
-                if record_c and candidate in self.opened(record_c, 1):
-                    verdict = rules.rule_path(entry, dest, rules.Opened(candidate, record_c["id"]))
-                    if verdict:
-                        self.settle(emulator, content, core, index, entry, [verdict])
-                        moved = True
-                        break
-            if not moved:
-                verdict = rules.rule_never_opened(entry, [record["id"], alone["id"]], content.name)
-                self.settle(emulator, content, core, index, entry, [verdict])
+    def judged(self, emulator: str, rid: str) -> bool:
+        """True for a record the boot oracle judged: only those prove a field."""
+        record = self.read_record(emulator, rid)
+        return bool(record) and record.get("record_version", 0) >= 4
 
     @staticmethod
     def opened(record: dict, position: int) -> set[str]:
@@ -1203,34 +1297,23 @@ class Campaign:
 
     def follow_up(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
                   digest: str, entry: dict, local: Path, dest: str, check: str) -> dict | None:
-        """The file staged at dest alone, against nothing there."""
-        key = self.check_key(core, content, check)
-        previous = self.state["checks"].get(unit.emulator, {}).get(key)
-        if not should_run(previous, core.sha256, digest):
-            return self.read_record(unit.emulator, (previous or {}).get("record"))
+        """The file staged at dest alone: does the core open it there."""
         stem = f"{core.name}-{slug(content.name)}"
-        argv = self.probe_args("check", core, unit.emulator, slug(f"{check}-{stem}"), content, load)
-        argv += frames + ["--vary", f"{dest}=", "--vary", f"{dest}={local}",
-                          "--observable", "frame_hashes", "--repeat", "1",
-                          "--claim", f"{core.name} opens {entry['name']} as {dest}",
-                          "--method", (f"{entry['name']} staged as {dest}, against nothing there, "
-                                       f"{content.frames} frames of {content.name}. The files the "
-                                       f"core opened are read from strace.")]
-        result = self.probe(argv, f"{unit.emulator}-{stem}-{slug(check)}")
-        self.remember(unit.emulator, key, result, core, digest)
-        return result.record
+        argv = self.probe_args("run", core, unit.emulator, slug(f"{check}-{stem}"), content, load)
+        argv += frames + ["--system-file", f"{dest}={local}"]
+        return self.check(unit.emulator, self.check_key(core, content, check), core, digest, argv,
+                          f"{unit.emulator}-{stem}-{slug(check)}")
 
     def follow_aliases(self, unit: Unit, core: Core, content: Content, load: Path, frames: list[str],
-                       digest: str, index: int, entry: dict, local: Path, record: dict,
-                       absent_label: str) -> None:
-        """Names the core looked for, with the file absent, that no entry declares."""
+                       digest: str, index: int, entry: dict, local: Path, none_record: dict) -> None:
+        """Names the core looked for, with no BIOS, that no entry declares."""
         declared = {e["name"] for e in unit.profile.get("files") or ()}
         declared |= {a for e in unit.profile.get("files") or () for a in e.get("aliases") or ()}
         suffix = Path(entry["name"]).suffix.casefold()
         dest = entry_dest(entry)
         folder = dest.rsplit("/", 1)[0] if "/" in dest else ""
         probed: list[str] = []
-        for rep in rules.variant(record, absent_label)["repetitions"]:
+        for rep in none_record["variants"][0]["repetitions"]:
             for path in rep.get("files_probed_absent") or ():
                 name = path.rsplit("/", 1)[-1]
                 where = path.rsplit("/", 1)[0] if "/" in path else ""
@@ -1240,7 +1323,7 @@ class Campaign:
         for candidate in probed[:MAX_ALIAS_CANDIDATES]:
             found = self.follow_up(unit, core, content, load, frames, digest, entry, local, candidate,
                                    f"alias:{entry['name']}:{candidate.rsplit('/', 1)[-1]}")
-            if found and candidate in self.opened(found, 1):
+            if found and candidate in self.opened(found, 0):
                 verdict = rules.rule_alias(self.reload_entry(unit, index), rules.Opened(candidate, found["id"]))
                 if verdict:
                     self.settle(unit.emulator, content, core, index, self.reload_entry(unit, index), [verdict])
@@ -1249,7 +1332,7 @@ class Campaign:
                      digest: str, index: int, entry: dict, dest: str, primary: Path,
                      variants: list[Path]) -> None:
         key = self.check_key(core, content, f"variants:{entry['name']}")
-        if not should_run(self.state["checks"].get(unit.emulator, {}).get(key), core.sha256, digest):
+        if not self.needed(unit.emulator, key, core, digest):
             return
         stem = f"{core.name}-{slug(content.name)}"
         argv = self.probe_args("check", core, unit.emulator, slug(f"variants-{entry['name']}-{stem}"),
@@ -1257,16 +1340,14 @@ class Campaign:
         for path in [primary, *variants]:
             argv += ["--vary", f"{dest}={path}"]
         argv += ["--observable", "frame_hashes", "--repeat", str(self.settings.check_repeat),
-                 "--claim", f"the copies of {entry['name']} change what {core.name} shows on {content.name}",
-                 "--method", (f"Each copy the collection holds staged as {dest}, each run "
-                              f"{self.settings.check_repeat} times for {content.frames} frames, "
-                              f"compared on the sampled frame hashes.")]
-        result = self.probe(argv, f"{unit.emulator}-{stem}-variants")
-        self.remember(unit.emulator, key, result, core, digest)
-        if result.record:
-            status = result.record["check"]["result"]["status"]
-            verdict = rules.rule_variants(status, [v["label"] for v in result.record["variants"]],
-                                          entry, result.record["id"])
+                 "--claim", f"every copy of {entry['name']} boots {content.name} on {core.name}",
+                 "--method", (f"Each copy the collection holds staged alone as {dest}, each run "
+                              f"{self.settings.check_repeat} times for {content.frames} frames. The "
+                              f"boot oracle judges each copy.")]
+        record = self.check(unit.emulator, key, core, digest, argv, f"{unit.emulator}-{stem}-variants")
+        if record:
+            verdict = rules.rule_variants([rules.run_of(record, v["label"]) for v in record["variants"]],
+                                          entry)
             if verdict:
                 self.settle(unit.emulator, content, core, index, entry, [verdict])
 
@@ -1291,9 +1372,10 @@ class Campaign:
             agrees = verdict.field in rules.SETTLED_FIELDS and rules.agrees(fresh, verdict)
             if verdict.apply:
                 updates: dict[str, object] = {}
-                if not agrees or verdict.field not in fresh:
+                if not agrees or (verdict.field not in fresh and verdict.field != "validation"):
                     updates[verdict.field] = verdict.value
-                evidence = rules.merge_evidence(fresh.get("evidence"), verdict)
+                evidence = rules.merge_evidence(fresh.get("evidence"), verdict,
+                                                keep=lambda rid: self.judged(emulator, rid))
                 if evidence != fresh.get("evidence"):
                     updates["evidence"] = evidence
                 if updates:
@@ -1331,7 +1413,8 @@ class Campaign:
         key = self.finding_key(emulator, content, entry, verdict)
         previous = self.latest_findings().get(key)
         if previous and previous["status"] == status and previous["rule"] == verdict.rule \
-                and previous["run_shows"] == verdict.value and previous["profile_says"] == says:
+                and previous["run_shows"] == verdict.value and previous["profile_says"] == says \
+                and previous["evidence"] == list(verdict.evidence):
             return
         line = mr.seal_line({
             "schema_version": 1, "found_at": now_iso(), "key": key, "emulator": emulator,
@@ -1364,6 +1447,60 @@ class Campaign:
                 return record["core"].get("library_version")
         return None
 
+    # -- reference frames -----------------------------------------------------------
+
+    def references(self, only: set[str], system: str | None, png_dir: Path) -> list[dict]:
+        """The full set run once per core and content, frames written as PNG,
+        and the screens that stay on long enough to serve as references."""
+        self.deadline = self.host.clock() + self.settings.budget
+        listed = []
+        for unit in self.units(only, system):
+            contents = self.unit_contents(unit, system)
+            if unit.pending or not contents:
+                continue
+            try:
+                cores = self.unit_cores(unit.emulator, unit.profile, unit.buildbot_only)
+            except Unavailable as exc:
+                self.log(f"{unit.emulator}: {exc}")
+                continue
+            for core in cores:
+                for content in contents:
+                    self.guard()
+                    self.store.hold(content, "references")
+                    try:
+                        load = load_file(self.store.get(content), content.load)
+                    except Unavailable as exc:
+                        self.log(f"{unit.emulator}: {exc}")
+                        continue
+                    stem = f"{core.name}-{slug(content.name)}"
+                    target = png_dir / slug(core.name) / slug(content.name)
+                    shutil.rmtree(target, ignore_errors=True)
+                    argv = self.probe_args("run", core, unit.emulator, slug(f"references-{stem}"),
+                                           content, load)
+                    argv += ["--frames", str(content.frames), "--frame-hash-every",
+                             str(content.frame_hash_every), "--repeat", str(self.settings.check_repeat),
+                             "--png-dir", str(target)]
+                    for index, entry in self.system_entries(unit.profile, content.system):
+                        local = self.local_file(entry)
+                        if local and f"{entry_dest(entry)}=" not in " ".join(argv):
+                            argv += ["--system-file", f"{entry_dest(entry)}={local}"]
+                    result = self.probe(argv, f"{unit.emulator}-{stem}-references")
+                    self.store.release(content, "references")
+                    if not result.record:
+                        self.log(f"{unit.emulator}: references of {content.name}: {result.reason}")
+                        continue
+                    record = result.record
+                    listed.append({
+                        "emulator": unit.emulator, "core": core.name,
+                        "library_name": record["core"]["library_name"], "content": content.name,
+                        "content_sha1": record["inputs"]["content"]["sha1"], "record": record["id"],
+                        "verdict": record["variants"][0]["verdict"],
+                        "png_dir": str(target / record["variants"][0]["label"]),
+                        "candidates": reference_candidates(record),
+                    })
+        save_state(self.paths.state, self.state)
+        return listed
+
     # -- the pass -----------------------------------------------------------------
 
     def units(self, only: set[str], system: str | None) -> list[Unit]:
@@ -1567,6 +1704,32 @@ class Campaign:
         return made
 
 
+def reference_candidates(record: dict, min_samples: int = 3) -> list[dict]:
+    """Screens of a record's first variant that are not one colour and stay
+    on for at least min_samples consecutive samples in every repetition:
+    what a reviewer looks at to choose reference frames."""
+    found: list[dict[str, tuple[int, int]]] = []
+    for rep in record["variants"][0]["repetitions"]:
+        observables = rep.get("observables") or {}
+        uniform = set(observables.get("uniform_frames") or ())
+        runs: dict[str, tuple[int, int]] = {}
+        previous, start, length = None, 0, 0
+        for frame, digest in [*(observables.get("frame_hashes") or ()), (None, None)]:
+            if digest == previous:
+                length += 1
+                continue
+            if previous and previous not in uniform and length >= min_samples:
+                first, longest = runs.get(previous, (start, 0))
+                runs[previous] = (min(first, start), max(longest, length))
+            previous, start, length = digest, frame, 1
+        found.append(runs)
+    common = set(found[0]) if found else set()
+    for runs in found[1:]:
+        common &= set(runs)
+    return sorted(({"sha1": digest, "first_frame": found[0][digest][0], "samples": found[0][digest][1]}
+                   for digest in common), key=lambda item: item["first_frame"])
+
+
 def reason_class(reason: str) -> str:
     """A pending reason without its detail, to count them."""
     head = reason.split(":", 1)[0].strip() if ":" in reason else reason
@@ -1661,9 +1824,17 @@ def main(argv: Sequence[str] | None = None) -> int:
     parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0],
                                      formatter_class=argparse.RawDescriptionHelpFormatter)
     sub = parser.add_subparsers(dest="command", required=True)
-    run_p = sub.add_parser("run", help="one pass: cores, content, checks, rules")
-    run_p.add_argument("--only", default="", help="emulator profiles or core names, comma-separated")
-    run_p.add_argument("--system", help="one system id")
+    scope = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
+    scope.add_argument("--only", default="", help="emulator profiles or core names, comma-separated")
+    scope.add_argument("--system", help="one system id")
+    scope.add_argument("--romfetch", default="",
+                       help=f"romfetch command (default: ${ROMFETCH_ENV}, else romfetch on PATH)")
+    scope.add_argument("--offline", action="store_true", help="no buildbot, no romfetch, no build")
+    scope.add_argument("--content-file", default=str(CONTENT_FILE),
+                       help="the test content list (default: tests/campaign/content.yml)")
+    scope.add_argument("--allow-unsandboxed", action="store_true",
+                       help="run cores and builds without bubblewrap when it is missing (refused by default)")
+    run_p = sub.add_parser("run", parents=[scope], help="one pass: cores, content, checks, rules")
     run_p.add_argument("--budget", type=parse_budget, default=6 * 3600.0,
                        help="stop starting checks after this long (6h, 90m, 3600s)")
     run_p.add_argument("--max-load-wait", type=float, default=3600,
@@ -1671,16 +1842,17 @@ def main(argv: Sequence[str] | None = None) -> int:
     run_p.add_argument("--min-free-gb", type=float, default=30, help="pause below this free space")
     run_p.add_argument("--cache-cap-gb", type=float, default=20,
                        help="content the campaign fetched is evicted past this size, oldest use first")
-    run_p.add_argument("--romfetch", default="",
-                       help=f"romfetch command (default: ${ROMFETCH_ENV}, else romfetch on PATH)")
-    run_p.add_argument("--offline", action="store_true", help="no buildbot, no romfetch, no build")
-    run_p.add_argument("--allow-unsandboxed", action="store_true",
-                       help="run cores and builds without bubblewrap when it is missing (refused by default)")
+    run_p.add_argument("--bench", action="store_true",
+                       help="also time each content that boots (benchmarks only choose between variants)")
     run_p.add_argument("--dry-run", action="store_true", help="print the plan, run nothing")
     run_p.add_argument("--commit", action="store_true",
                        help="commit records, findings and each changed profile behind the gate")
     run_p.add_argument("--commit-every", type=int, default=0, metavar="N",
                        help="with --commit, also commit after every N emulators")
+    refs_p = sub.add_parser("references", parents=[scope],
+                            help="run the full set with PNG frames and list reference candidates")
+    refs_p.add_argument("--png-dir", default=str(ROOT / "tmp" / "campaign" / "references"),
+                        help="where the frames are written, one folder per core and content")
     status_p = sub.add_parser("status", help="done, failed and pending checks, and why")
     status_p.add_argument("--json", action="store_true", help="the report as JSON")
     findings_p = sub.add_parser("findings", help="profile claims the runs contradict, open for review")
@@ -1688,9 +1860,9 @@ def main(argv: Sequence[str] | None = None) -> int:
     findings_p.add_argument("--json", action="store_true", help="the findings as JSON")
     sub.add_parser("gate", help="tests, schemas, verify and pack of the changed profiles")
     timer_p = sub.add_parser("install-timer",
-                             help="write the autorun units, the campaign job's timer among them")
+                             help="opt-in: write the autorun units, the campaign job's timer among them")
     timer_p.add_argument("--enable", action="store_true",
-                         help="also reload systemd and enable the timers")
+                         help="also reload systemd and enable the timers (never by default)")
     args = parser.parse_args(argv)
 
     paths = Paths.default()
@@ -1713,20 +1885,34 @@ def main(argv: Sequence[str] | None = None) -> int:
         settings = Settings(budget=args.budget, max_load_wait=args.max_load_wait,
                             min_free_gb=args.min_free_gb, cache_cap_gb=args.cache_cap_gb,
                             romfetch=args.romfetch, network=not args.offline,
+                            allow_unsandboxed=args.allow_unsandboxed, bench=args.bench)
+    elif args.command == "references":
+        settings = Settings(romfetch=args.romfetch, network=not args.offline,
                             allow_unsandboxed=args.allow_unsandboxed)
+    if args.command in ("run", "references"):
+        paths.content_file = Path(args.content_file)
     campaign = Campaign(paths, settings)
     if args.command == "status":
         report = status_report(campaign)
         print(json.dumps(report, indent=2) if args.json else render_status(report))
         return 0
     only = {item.strip() for item in args.only.split(",") if item.strip()}
+    if args.command == "references":
+        try:
+            with host_lock(campaign.lock_path):
+                listed = campaign.references(only, args.system, Path(args.png_dir))
+        except autorun.LockBusy as exc:
+            print(f"campaign: deferred, {exc}", file=sys.stderr)
+            return 75
+        print(json.dumps(listed, indent=2))
+        return 0
     if args.dry_run:
         campaign.run(only, args.system, dry_run=True)
         return 0
     campaign.dirty_at_start = set(autorun.changed_files(paths.root)) if args.commit else set()
 
     def stop(signum: int, frame: object) -> None:
-        # A timer's TimeoutStartSec ends the pass like Ctrl-C: state saved, pass line written.
+        # SIGTERM ends the pass like Ctrl-C: state saved, pass line written.
         raise KeyboardInterrupt
     signal.signal(signal.SIGTERM, stop)
     try:

scripts/campaign_profiles.py

@@ -23,6 +23,7 @@ ANCHORS = {
     "hle_fallback": ("required", "name"),
     "path": ("name",),
     "aliases": ("path", "name"),
+    "validation": ("sha1", "md5", "crc32", "size", "name"),
 }
 
 

scripts/campaign_rules.py

@@ -1,25 +1,36 @@
-"""The campaign's fixed rule table: which observable settles which profile field.
+"""The campaign's fixed rule table: which verdicts settle which profile field.
 
-Each rule is a pure function of a core probe record (or what was read from
-one) and a profile entry. It returns verdicts. A verdict with `apply` set is
-settled by the table and written into the profile with the record ids as
-`evidence:`. Any other verdict goes to the review queue
-(measurements/findings.jsonl).
+Each rule is a pure function of verdicts read from core probe records (the
+boot oracle of core_probe.py: boots, bios_error, crash, hang, unknown) and
+a profile entry. It returns verdicts on profile fields. One with `apply` set
+is settled by the table and written into the profile with the record ids as
+`evidence:`. Any other goes to the review queue (measurements/findings.jsonl).
 
-  observable                                        field          settled
-  boots to the game with no file                    hle_fallback   true
-  no game screen without it, the game with it       required       true
-                                                    hle_fallback   false
-  opens the staged file under another path          path           that path
-  opens the staged file under another name          aliases        name added
-  a staged file the content should use, never read  -              flagged only
+Three runs are compared for a file, all on the same content: `none` with no
+BIOS at all, `present` with the file alone, and wrong files staged under its
+name (zeros of its size, its first half, another region's dump).
 
-"Boots to the game" is read from the sampled frames: the run without the
-file shows at least MIN_SHARED_SCREENS distinct screens that the run with
-the file also shows, single-colour frames excluded. "No game screen" is a
-run that did not load, or whose every sampled frame is a single colour. A
-run that shows screens of its own that the reference never shows settles
-nothing.
+  runs                                          field          settled
+  none boots, the file changes the run          hle_fallback   true
+  none fails, present boots                     required       true
+                                                hle_fallback   false
+  zeros refused, present boots                  validation     a content check
+  zeros used, present boots                     validation     no content check
+  half refused, zeros used                      validation     size
+  half used                                     validation     no size check
+  opens the staged file under another path      path           that path
+  opens the staged file under another name      aliases        name added
+  a staged file the content should use, never   -              flagged only
+  read
+  copies of one file that boot and fail         -              queued
+
+"Fails" is bios_error, crash or hang. A wrong file is refused when the
+core reports a BIOS error, or when it boots without the BIOS screens the
+right file shows (it fell back to a built-in BIOS). It is used when it
+turns the run into a hang or a crash that the run without any file does not
+show. Anything else settles nothing. Of validation, the table writes only a
+missing `size`. A contradicted content check goes to review: which hash a
+core computes is read in its code.
 
 Standard library only.
 """
@@ -29,23 +40,34 @@ from __future__ import annotations
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass, field
 
-MIN_SHARED_SCREENS = 2
 BOOTS = "boots"
-FAILS = "fails"
-INCONCLUSIVE = "inconclusive"
+BIOS_ERROR = "bios_error"
+CRASH = "crash"
+HANG = "hang"
+UNKNOWN = "unknown"
+FAILS = (BIOS_ERROR, CRASH, HANG)
+REFUSED = "refused"
+USED = "used"
+CONTENT_CHECKS = ("md5", "sha1", "sha256", "crc32", "adler32")
 
 # A field an entry leaves out reads as this value.
-FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": []}
-SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases")
+FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": [],
+                                     "validation": []}
+SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases", "validation")
 
 
 @dataclass(frozen=True)
-class Boot:
-    """What the runs without the file showed, against the runs with it."""
+class Run:
+    """The verdict of one variant of a record: what a rule reads."""
 
-    status: str
-    detail: str
-    shared: int = 0
+    verdict: str
+    record_id: str
+    label: str
+    stages: tuple[str, ...] = ()
+
+    @property
+    def shows_bios(self) -> bool:
+        return "bios" in self.stages
 
 
 @dataclass(frozen=True)
@@ -67,13 +89,6 @@ class Opened:
     extra: dict = field(default_factory=dict)
 
 
-def screens(repetition: dict) -> set[str]:
-    """The distinct sampled frames of a repetition that are not one colour."""
-    observables = repetition.get("observables") or {}
-    uniform = set(observables.get("uniform_frames") or ())
-    return {digest for _, digest in observables.get("frame_hashes") or () if digest not in uniform}
-
-
 def variant(record: dict, label: str) -> dict:
     for item in record.get("variants") or ():
         if item.get("label") == label:
@@ -81,43 +96,16 @@ def variant(record: dict, label: str) -> dict:
     raise KeyError(f"record {record.get('id')}: no variant {label!r}")
 
 
-def boot_outcome(record: dict, absent_label: str, present_label: str) -> Boot:
-    """Whether the core reached the game without the file.
+def run_of(record: dict, label: str) -> Run:
+    """A variant's verdict as a rule reads it. A record from before the
+    oracle has none, and reads as unknown."""
+    item = variant(record, label)
+    evidence = item.get("verdict_evidence") or {}
+    return Run(item.get("verdict", UNKNOWN), record["id"], label, tuple(evidence.get("stages") or ()))
 
-    Every repetition must agree: one that boots and one that does not leave
-    the question open.
-    """
-    present = variant(record, present_label)["repetitions"]
-    if any(not rep.get("loaded") for rep in present):
-        return Boot(INCONCLUSIVE, "with the file, the core did not load the content")
-    reference: set[str] = set()
-    for rep in present:
-        shown = screens(rep)
-        if not shown:
-            return Boot(INCONCLUSIVE, "with the file, the core showed no game screen")
-        reference |= shown
-    outcomes: list[str] = []
-    shared_counts: list[int] = []
-    for rep in variant(record, absent_label)["repetitions"]:
-        if not rep.get("loaded"):
-            outcomes.append(FAILS)
-            continue
-        shown = screens(rep)
-        if not shown:
-            outcomes.append(FAILS)
-            continue
-        shared = len(shown & reference)
-        shared_counts.append(shared)
-        outcomes.append(BOOTS if shared >= MIN_SHARED_SCREENS else INCONCLUSIVE)
-    shared = min(shared_counts) if shared_counts else 0
-    if all(o == BOOTS for o in outcomes):
-        return Boot(BOOTS, f"without the file, every run showed {shared} or more screens the "
-                           f"run with the file shows", shared)
-    if all(o == FAILS for o in outcomes):
-        return Boot(FAILS, "without the file, no run loaded the content or showed more than "
-                           "one colour, while the run with the file reached the game")
-    return Boot(INCONCLUSIVE, "without the file, the runs showed screens the run with the file "
-                              "never shows, or disagreed with each other", shared)
+
+def ids(*runs: Run) -> tuple[str, ...]:
+    return tuple(dict.fromkeys(run.record_id for run in runs))
 
 
 def current(entry: dict, name: str) -> object:
@@ -130,32 +118,123 @@ def agrees(entry: dict, verdict: Verdict) -> bool:
     """True when the profile already says what the run showed."""
     if verdict.field == "aliases":
         return set(verdict.value or ()) <= set(entry.get("aliases") or ())
+    if verdict.field == "validation":
+        return verdict.rule == "validation_agrees"
     return current(entry, verdict.field) == verdict.value
 
 
-def rule_bios_need(boot: Boot, entry: dict, record_id: str) -> list[Verdict]:
-    """Boot with no file against boot with the declared file."""
+def read(none: Run, *runs: Run) -> bool:
+    """True when the file changes the run: alone or as a wrong file, one
+    variant ends otherwise than the run with no BIOS, or reaches other
+    stages. A core can open a file without using it for this content (a
+    scan that hashes every file of the folder)."""
+    return any(run.verdict != none.verdict or run.stages != none.stages for run in runs if run)
+
+
+def rule_need(none: Run, present: Run, entry: dict, used: bool = True) -> list[Verdict]:
+    """No BIOS at all against the file alone: required and hle_fallback. A
+    file the content never uses settles nothing."""
     name = entry["name"]
-    evidence = (record_id,)
-    if boot.status == BOOTS:
+    if none.verdict == BOOTS and not used:
+        return []
+    if none.verdict == BOOTS:
         out = [Verdict("boots_without_file", "hle_fallback", True,
-                       f"the core reaches the game without {name}: {boot.detail}", evidence, True)]
+                       f"the core boots the content with no BIOS at all, {name} included",
+                       ids(none), True)]
         if entry.get("required") is True:
             out.append(Verdict(
                 "boots_without_file", "required", False,
-                f"the profile says {name} is required, and the core reaches the game "
-                f"without it. The table settles hle_fallback from this run, not required",
-                evidence))
+                f"the profile says {name} is required, and the core boots without it. The table "
+                f"settles hle_fallback from this run, not required", ids(none)))
         return out
-    if boot.status == FAILS:
+    if none.verdict in FAILS and present.verdict == BOOTS:
         return [
             Verdict("fails_without_file", "required", True,
-                    f"the core reaches the game only with {name}: {boot.detail}", evidence, True),
+                    f"with no BIOS the run ends in {none.verdict}, with {name} alone it boots",
+                    ids(none, present), True),
             Verdict("fails_without_file", "hle_fallback", False,
-                    f"no stand-in for {name}: {boot.detail}", evidence, True),
+                    f"no stand-in for {name}: with no BIOS the run ends in {none.verdict}",
+                    ids(none, present), True),
         ]
-    return [Verdict("inconclusive", "hle_fallback", None,
-                    f"the runs with and without {name} settle nothing: {boot.detail}", evidence)]
+    return [Verdict("unsettled", "hle_fallback", None,
+                    f"with no BIOS the run is {none.verdict} and with {name} alone {present.verdict}: "
+                    f"no rule settles a field from that", ids(none, present))]
+
+
+def file_use(wrong: Run, none: Run, present: Run) -> str | None:
+    """Whether the core refused a wrong file or ran it, or None."""
+    if present.verdict != BOOTS:
+        return None
+    if wrong.verdict == BIOS_ERROR:
+        return REFUSED
+    if wrong.verdict == BOOTS:
+        if wrong.shows_bios:
+            return USED
+        return REFUSED if present.shows_bios else None
+    if wrong.verdict in (HANG, CRASH) and none.verdict in (BOOTS, BIOS_ERROR):
+        return USED
+    return None
+
+
+def declared_checks(entry: dict) -> tuple[set[str], bool]:
+    """The checks the entry's validation names, and whether it is one list
+    (a per-build dict is never edited by the table)."""
+    value = entry.get("validation") or []
+    if isinstance(value, dict):
+        return {str(v) for items in value.values() for v in items or ()}, False
+    return {str(v) for v in value}, True
+
+
+def rule_validation(zeros: Run | None, half: Run | None, none: Run, present: Run,
+                    entry: dict) -> Verdict | None:
+    """Same-size and short wrong files against the right one: validation."""
+    name = entry["name"]
+    content = {REFUSED: True, USED: False}.get(file_use(zeros, none, present)) if zeros else None
+    half_use = file_use(half, none, present) if half else None
+    size: bool | None = None
+    if half_use == USED:
+        size, content = False, False if content is None else content
+    elif half_use == REFUSED and content is False:
+        size = True
+    if content is None and size is None:
+        return None
+    declared, editable = declared_checks(entry)
+    evidence = ids(*(run for run in (none, present, zeros, half) if run))
+    said = sorted(declared)
+    shown = []
+    problems = []
+    if content is not None:
+        shown.append("checks the content" if content else "does not check the content")
+        if content != bool(declared & set(CONTENT_CHECKS)):
+            problems.append("a same-size file of zeros is " + ("refused" if content else "run")
+                            + f", the profile declares validation {said}")
+    missing_size = size is True and "size" not in declared
+    if size is not None:
+        shown.append("checks the size" if size else "does not check the size")
+        if size is False and "size" in declared:
+            problems.append(f"the first half of {name} is run, the profile declares a size check")
+    statement = f"the core {' and '.join(shown)} of {name}"
+    # What the runs show, for a finding beside what the profile says.
+    checks = {"content": content, "size": size}
+    if problems:
+        return Verdict("validation_contradicted", "validation", checks,
+                       f"{statement}: {'; '.join(problems)}", evidence)
+    if missing_size:
+        if not editable:
+            return Verdict("validation_contradicted", "validation", checks,
+                           f"{statement}: the per-build validation has no size check", evidence)
+        value = [*(entry.get("validation") or []), "size"]
+        return Verdict("validation_size_added", "validation", value,
+                       f"{statement}: half of the file is refused while zeros of its size are run",
+                       evidence, True)
+    return Verdict("validation_agrees", "validation", entry.get("validation") or [], statement,
+                   evidence, editable)
+
+
+def rule_region(dump: Run, present: Run, entry: dict, game_region: str | None) -> str:
+    """What the core did with another region's dump under this file's name."""
+    return (f"{dump.label} staged as {entry['name']} with a {game_region or 'region-less'} game: "
+            f"{dump.verdict}, {entry['name']} itself: {present.verdict}")
 
 
 def rule_path(entry: dict, declared: str, opened: Opened) -> Verdict | None:
@@ -184,20 +263,25 @@ def rule_never_opened(entry: dict, record_ids: Sequence[str], content: str) -> V
                    f"{content}", tuple(record_ids))
 
 
-def rule_variants(status: str, labels: Iterable[str], entry: dict, record_id: str) -> Verdict | None:
-    """Copies of one file that change what the core shows: a pack choice to review."""
-    if status != "differs":
+def rule_variants(runs: Iterable[Run], entry: dict) -> Verdict | None:
+    """Copies of one file whose verdicts differ: a pack choice to review."""
+    runs = list(runs)
+    if len({run.verdict for run in runs}) < 2:
         return None
-    return Verdict("variant_changes_output", "variants", sorted(labels),
-                   f"the copies of {entry['name']} change what the core shows",
-                   (record_id,))
+    shown = ", ".join(f"{run.label} {run.verdict}" for run in runs)
+    return Verdict("variant_changes_verdict", "variants", sorted(run.label for run in runs),
+                   f"the copies of {entry['name']} do not boot alike: {shown}", ids(*runs))
 
 
-def merge_evidence(existing: dict | None, verdict: Verdict) -> dict:
-    """The entry's evidence: map with the verdict's record ids added under its field."""
+def merge_evidence(existing: dict | None, verdict: Verdict,
+                   keep: object = lambda rid: True) -> dict:
+    """The entry's evidence: map with the verdict's record ids added under its
+    field. An id `keep` refuses is dropped from that field: a record from
+    before the oracle no longer proves anything."""
     out = {key: list(value) for key, value in (existing or {}).items()}
-    ids = out.setdefault(verdict.field, [])
+    ids_ = [rid for rid in out.get(verdict.field, []) if keep(rid)]
     for rid in verdict.evidence:
-        if rid not in ids:
-            ids.append(rid)
-    return {key: out[key] for key in sorted(out)}
+        if rid not in ids_:
+            ids_.append(rid)
+    out[verdict.field] = ids_
+    return {key: out[key] for key in sorted(out) if out[key]}

scripts/core_probe.py

@@ -21,6 +21,7 @@ import argparse
 import contextlib
 import ctypes
 import datetime as dt
+import fcntl
 import hashlib
 import json
 import os
@@ -35,7 +36,7 @@ import sys
 import tempfile
 import time
 import zlib
-from collections.abc import Iterable, Sequence
+from collections.abc import Iterable, Iterator, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -73,6 +74,39 @@ PRINTF_SPEC = re.compile(r"%(?P<flags>[-+ #0]*)(?P<width>\*|\d+)?(?:\.(?P<prec>\
                          r"(?P<len>hh|h|ll|l|j|z|t|L)?(?P<conv>[diouxXeEfFgGaAcspn%])")
 
 
+# One measurement at a time on the host: each core_probe execution takes this
+# lock itself, its load check included, and releases it when it ends. A pass
+# of many runs never holds it between them, so two passes interleave.
+PROBE_LOCK_ENV = "RETROBIOS_PROBE_LOCK"
+PROBE_LOCK_HELD_ENV = "RETROBIOS_PROBE_LOCK_HELD"
+DEFAULT_PROBE_LOCK = Path.home() / ".local" / "state" / "retrobios" / "probe.lock"
+
+
+def probe_lock_path() -> Path:
+    return Path(os.environ.get(PROBE_LOCK_ENV) or DEFAULT_PROBE_LOCK)
+
+
+@contextlib.contextmanager
+def host_probe_lock() -> Iterator[None]:
+    """The host's probe lock for one execution. A caller that already holds
+    it for this run says so with RETROBIOS_PROBE_LOCK_HELD."""
+    if os.environ.get(PROBE_LOCK_HELD_ENV):
+        yield
+        return
+    path = probe_lock_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a") as handle:
+        try:
+            fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            print(f"waiting for {path}: another measurement is running", file=sys.stderr)
+            fcntl.flock(handle, fcntl.LOCK_EX)
+        try:
+            yield
+        finally:
+            fcntl.flock(handle, fcntl.LOCK_UN)
+
+
 class ProbeError(RuntimeError):
     """A run that cannot produce a trustworthy result."""
 
@@ -375,15 +409,25 @@ class Frontend:
         self.unanswered: dict[int, int] = {}
         self.input_queries = 0
         self.input_pressed = 0
+        # What a run showed so far, one JSON line per event, flushed at once:
+        # a core that kills the worker leaves its logs and frames behind.
+        events = spec.get("events")
+        self.events = open(events, "a", encoding="utf-8", buffering=1) if events else None  # noqa: SIM115
         self.callbacks = (
             EnvFn(self.environment), VideoFn(self.video_refresh), AudioFn(self.audio_sample),
             AudioBatchFn(self.audio_batch), PollFn(lambda: None), InputFn(self.input_state),
         )
         self.log_callback = RetroLogCallback(LogFn(self.log))
 
+    def event(self, kind: str, **values: object) -> None:
+        if self.events:
+            self.events.write(json.dumps({"event": kind, **values}) + "\n")
+
     def log(self, level: int, fmt: bytes, *args: int) -> None:
         name = LOG_LEVELS[level] if 0 <= level < len(LOG_LEVELS) else str(level)
-        self.logs.append({"frame": self.frame, "level": name, "text": render_printf(fmt or b"", args)})
+        line = {"frame": self.frame, "level": name, "text": render_printf(fmt or b"", args)}
+        self.logs.append(line)
+        self.event("log", **line)
 
     def environment(self, cmd: int, data: int) -> bool:
         cmd &= ~ENV_EXPERIMENTAL
@@ -446,8 +490,10 @@ class Frontend:
         if every and self.frame % every == 0:
             digest = hashlib.sha1(self.last_frame).hexdigest()
             self.frame_hashes.append([self.frame, digest])
-            if digest not in self.uniform and uniform_frame(self.last_frame, PIXEL_BYTES[self.pixel_format]):
+            flat = digest in self.uniform or uniform_frame(self.last_frame, PIXEL_BYTES[self.pixel_format])
+            if flat:
                 self.uniform.add(digest)
+            self.event("frame", frame=self.frame, sha1=digest, uniform=flat)
             png_dir = self.spec.get("png_dir")
             if png_dir and self.last_frame:
                 rgb = to_rgb(self.pixel_format, *self.last_size, self.last_frame)
@@ -483,6 +529,7 @@ def run_worker(spec: dict) -> dict:
         getattr(core, name)(fn)
     info = RetroSystemInfo()
     core.retro_get_system_info(ctypes.byref(info))
+    front.event("info", library_name=info.library_name.decode(), library_version=info.library_version.decode())
     core.retro_init()
     game = None
     if spec.get("content"):
@@ -494,7 +541,9 @@ def run_worker(spec: dict) -> dict:
             front.option_buffers.append(holder)
             game.data, game.size = ctypes.cast(holder, ctypes.c_void_p), len(blob)
     core.retro_load_game.restype = ctypes.c_bool
-    if not core.retro_load_game(ctypes.byref(game) if game else None):
+    loaded = bool(core.retro_load_game(ctypes.byref(game) if game else None))
+    front.event("loaded", value=loaded)
+    if not loaded:
         return {"loaded": False, "logs": front.logs, "library_name": info.library_name.decode()}
     # A frontend plugs a joypad into each port after loading, as RetroArch does.
     for port in range(spec["joypad_ports"]):
@@ -619,10 +668,41 @@ def core_identity(args: argparse.Namespace) -> dict:
 # Runs (parent side)
 # --------------------------------------------------------------------------
 
+@dataclass(frozen=True)
+class Synthetic:
+    """A wrong file made for a negative test: `@zeros:SIZE` is SIZE zero
+    bytes, `@head:SIZE:PATH` the first SIZE bytes of PATH."""
+
+    kind: str
+    size: int
+    source: Path | None = None
+
+    @classmethod
+    def parse(cls, text: str) -> Synthetic:
+        kind, _, rest = text[1:].partition(":")
+        size_text, _, source = rest.partition(":")
+        if kind not in ("zeros", "head") or not size_text.isdigit() or (kind == "head") != bool(source):
+            raise argparse.ArgumentTypeError(
+                f"{text!r}: a made file is @zeros:SIZE or @head:SIZE:PATH")
+        return cls(kind, int(size_text), Path(source) if source else None)
+
+    def label(self, dest: str) -> str:
+        if self.kind == "zeros":
+            return f"{dest}-zeros-{self.size}"
+        return f"{dest}-head-{self.size}-of-{self.source.name}"
+
+    def write(self, out: Path) -> None:
+        if self.kind == "zeros":
+            out.write_bytes(bytes(self.size))
+            return
+        with self.source.open("rb") as handle:
+            out.write_bytes(handle.read(self.size))
+
+
 @dataclass
 class Variant:
     label: str
-    files: dict[str, Path | None] = field(default_factory=dict)
+    files: dict[str, Path | Synthetic | None] = field(default_factory=dict)
 
 
 def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> dict[str, str | None]:
@@ -632,10 +712,14 @@ def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> d
     for dest, src in {**base, **variant.files}.items():
         out = target / dest
         if src is None:
+            out.unlink(missing_ok=True)
             staged[dest] = None
             continue
         out.parent.mkdir(parents=True, exist_ok=True)
-        shutil.copyfile(src, out)
+        if isinstance(src, Synthetic):
+            src.write(out)
+        else:
+            shutil.copyfile(src, out)
         staged[dest] = file_digest(out)
     return staged
 
@@ -781,6 +865,45 @@ def run_group(command: list[str], timeout: float, env: dict[str, str]) -> subpro
     return subprocess.CompletedProcess(command, proc.returncode, out, err)
 
 
+WORKER_TRACEBACK = "Traceback (most recent call last)"
+
+
+def exit_description(returncode: int) -> dict:
+    """The status of a worker that died. A signal reaches the probe as a
+    negative code, or as 128 + its number through bubblewrap's init."""
+    number = -returncode if returncode < 0 else returncode - 128 if 128 < returncode < 160 else 0
+    name = None
+    if number:
+        with contextlib.suppress(ValueError):
+            name = signal.Signals(number).name
+    return {"status": returncode, "signal": name}
+
+
+def partial_result(events: Path, spec: dict) -> dict:
+    """A repetition rebuilt from the events a worker wrote before it died."""
+    lines = []
+    if events.is_file():
+        for line in events.read_text(encoding="utf-8", errors="replace").splitlines():
+            with contextlib.suppress(ValueError):
+                lines.append(json.loads(line))
+    info = next((e for e in lines if e.get("event") == "info"), {})
+    loaded = next((e["value"] for e in lines if e.get("event") == "loaded"), False)
+    frames = [e for e in lines if e.get("event") == "frame"]
+    result: dict = {
+        "loaded": bool(loaded),
+        "library_name": info.get("library_name", ""),
+        "logs": [{k: e[k] for k in ("frame", "level", "text")} for e in lines if e.get("event") == "log"],
+    }
+    if info.get("library_version"):
+        result["library_version"] = info["library_version"]
+    if spec["frame_hash_every"]:
+        result["observables"] = {
+            "frame_hashes": [[e["frame"], e["sha1"]] for e in frames],
+            "uniform_frames": sorted({e["sha1"] for e in frames if e.get("uniform")}),
+        }
+    return result
+
+
 def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None,
              png_dir: Path | None) -> dict:
     save_dir = scratch / "save"
@@ -800,6 +923,7 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou
         "png_dir": str(png_dir) if png_dir else None,
         "state_out": str(state_out) if state_out else None,
         "env": dict(args.env),
+        "events": str((scratch / "events.jsonl").resolve()),
     }
     spec_path, out_path, trace_path = scratch / "spec.json", scratch / "result.json", scratch / "strace.txt"
     spec_path.write_text(json.dumps(spec), encoding="utf-8")
@@ -816,10 +940,20 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou
     if args.require_sandbox and isolation != SANDBOX:
         raise ProbeError(f"refused to run the core outside a sandbox: {isolation}")
     # The core sees the variables it was given and nothing of the host's.
-    proc = run_group(prefix + command, args.timeout, dict(args.env))
-    if proc.returncode != 0 or not out_path.exists():
-        raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}")
-    result = json.loads(out_path.read_text(encoding="utf-8"))
+    try:
+        proc = run_group(prefix + command, args.timeout, dict(args.env))
+    except subprocess.TimeoutExpired:
+        result = partial_result(scratch / "events.jsonl", spec)
+        result["exit"] = {"status": None, "signal": "SIGKILL", "timeout_s": args.timeout}
+    else:
+        if proc.returncode == 0 and out_path.exists():
+            result = json.loads(out_path.read_text(encoding="utf-8"))
+        elif WORKER_TRACEBACK in proc.stderr:
+            # The probe's own code failed: no verdict on the core is drawn.
+            raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}")
+        else:
+            result = partial_result(scratch / "events.jsonl", spec)
+            result["exit"] = exit_description(proc.returncode)
     places = [(str(system_dir.resolve()), "<system>"), (str(save_dir.resolve()), "<save>")]
     if args.content:
         places.append((str(Path(args.content).resolve().parent), "<content>"))
@@ -872,6 +1006,10 @@ def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Varian
         if state_out:
             state_out.parent.mkdir(parents=True, exist_ok=True)
         rep = run_once(args, system_dir, scratch, state_out, png_dir)
+        oracle = oracle_for(args.oracle_data, args.content_sha1, rep.get("library_name"))
+        rep["verdict"], rep["verdict_evidence"] = judge(rep, oracle)
+        if rep.get("exit"):
+            flags.append(f"repetition {index}: {rep['verdict_evidence']['detail']}")
         if not rep.get("loaded"):
             flags.append(f"repetition {index}: retro_load_game returned false")
         for entry in rep.get("files_opened") or []:
@@ -885,11 +1023,14 @@ def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Varian
         if rep.get("observables") != reference:
             differing = sorted(k for k in (reference or {}) if (rep.get("observables") or {}).get(k) != reference[k])
             flags.append(f"repetition {index} disagrees with repetition 0 on {', '.join(differing)}")
+    verdict, evidence = variant_verdict(reps)
     return {
         "label": variant.label,
         "system_files": {dest: sha for dest, sha in sorted(staged.items())},
         "repetitions": reps,
         "flags": flags,
+        "verdict": verdict,
+        "verdict_evidence": evidence,
     }
 
 
@@ -946,12 +1087,18 @@ def build_parser() -> argparse.ArgumentParser:
                         help="refuse to record when the 1-minute load average exceeds this "
                              "(default: a quarter of the CPU count)")
     common.add_argument("--work-dir", default=str(DEFAULT_WORK_DIR), help="scratch directory")
-    common.add_argument("--timeout", type=int, default=1800, help="seconds per repetition")
+    common.add_argument("--timeout", type=int, default=1800,
+                        help="seconds per repetition, after which the run is killed and judged a hang")
+    common.add_argument("--oracle", default=None, metavar="FILE",
+                        help="reference frames and error patterns the verdicts are judged against "
+                             "(default: tests/campaign/oracle.yml)")
     common.add_argument("--no-trace", action="store_true", help="do not run the worker under strace")
     common.add_argument("--require-sandbox", action="store_true",
                         help="refuse to run the core unless bubblewrap can isolate it")
 
-    sub.add_parser("run", parents=[common], help="one execution")
+    run = sub.add_parser("run", parents=[common], help="one execution")
+    run.add_argument("--repeat", type=int, default=1,
+                     help="runs, each in a fresh process: the verdict is the one they all agree on")
     bench = sub.add_parser("bench", parents=[common], help="timed repetitions")
     bench.add_argument("--repeat", type=int, default=MIN_BENCH_REPETITIONS,
                        help=f"timed repetitions, each in a fresh process (at least {MIN_BENCH_REPETITIONS} to record)")
@@ -993,12 +1140,16 @@ def validate_args(args: argparse.Namespace) -> list[str]:
     for dest, src in args.system_file:
         if not Path(src).is_file():
             problems.append(f"--system-file {dest}={src}: not a file")
+    if args.repeat < 1:
+        problems.append("--repeat must be >= 1")
     if args.frames < 1 or args.warmup < 0:
         problems.append("--frames must be >= 1 and --warmup >= 0")
     if args.require_sandbox:
         isolation = sandbox_argv([])[1]
         if isolation != SANDBOX:
             problems.append(f"--require-sandbox: {isolation}")
+    if args.oracle and not Path(args.oracle).is_file():
+        problems.append(f"--oracle {args.oracle}: not a file")
     if args.png_dir and not args.frame_hash_every:
         problems.append("--png-dir needs --frame-hash-every")
     if args.record:
@@ -1022,7 +1173,15 @@ def validate_args(args: argparse.Namespace) -> list[str]:
         if len(dests) != 1:
             problems.append("--vary must name the same DEST in every occurrence")
         for dest, src in args.vary:
-            if src and not Path(src).is_file():
+            if src.startswith("@"):
+                try:
+                    made = Synthetic.parse(src)
+                except argparse.ArgumentTypeError as exc:
+                    problems.append(f"--vary {dest}: {exc}")
+                    continue
+                if made.source and not made.source.is_file():
+                    problems.append(f"--vary {dest}={src}: {made.source} is not a file")
+            elif src and not Path(src).is_file():
                 problems.append(f"--vary {dest}={src}: not a file")
         names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var}
         if args.frame_hash_every:
@@ -1038,11 +1197,12 @@ def variants_for(args: argparse.Namespace) -> list[Variant]:
     out: list[Variant] = []
     seen: set[str] = set()
     for dest, src in args.vary:
-        label = Path(src).name if src else f"{dest}-absent"
+        made = Synthetic.parse(src) if src.startswith("@") else None
+        label = made.label(dest) if made else Path(src).name if src else f"{dest}-absent"
         while label in seen:
             label += "+"
         seen.add(label)
-        out.append(Variant(label, {dest: Path(src) if src else None}))
+        out.append(Variant(label, {dest: made or (Path(src) if src else None)}))
     return out
 
 
@@ -1088,6 +1248,144 @@ def check_result(variants: list[dict], observable: str) -> dict:
     }
 
 
+# --------------------------------------------------------------------------
+# Boot oracle: what a run reached, from positive evidence
+# --------------------------------------------------------------------------
+
+BOOTS, BIOS_ERROR, CRASH, HANG, UNKNOWN = "boots", "bios_error", "crash", "hang", "unknown"
+VERDICTS = (BOOTS, BIOS_ERROR, CRASH, HANG, UNKNOWN)
+DEFAULT_ORACLE = ROOT / "tests" / "campaign" / "oracle.yml"
+
+
+def oracle_for(data: dict, content_sha1: str | None, library_name: str | None) -> dict:
+    """The oracle entries that judge one content on one core: its reference
+    frames, the error patterns every core shares and the core's own, the
+    core's known error screens, and the log lines by which it says it loaded
+    the console's own BIOS."""
+    core = (data.get("cores") or {}).get(library_name or "") or {}
+    references: list[dict] = []
+    for item in data.get("content") or ():
+        if content_sha1 and item.get("sha1") == content_sha1:
+            block = (item.get("references") or {}).get(library_name or "") or {}
+            references = [{"sha1": f["sha1"], "stage": f["stage"], "shows": f["shows"]}
+                          for f in block.get("frames") or ()]
+    patterns = [p["pattern"] for p in (data.get("error_patterns") or ())]
+    patterns += [p["pattern"] for p in core.get("error_patterns") or ()]
+    screens = [{"sha1": s["sha1"], "shows": s["shows"]} for s in core.get("error_screens") or ()]
+    loaded = [p["pattern"] for p in core.get("bios_patterns") or ()]
+    return {"references": references, "error_patterns": patterns, "error_screens": screens,
+            "bios_patterns": loaded}
+
+
+def sampled(rep: dict) -> list[tuple[int, str]]:
+    observables = rep.get("observables") or {}
+    return [(int(frame), digest) for frame, digest in observables.get("frame_hashes") or ()]
+
+
+def judge(rep: dict, oracle: dict) -> tuple[str, dict]:
+    """The verdict of one repetition and the evidence that decided it.
+
+    In order: a worker killed after the timeout is a hang and one that died
+    a crash, from the process status. A reference frame of the game reached
+    is a boot: the console's own screens alone are not. Without one, an error pattern of the core log or a known error screen
+    is a BIOS error. Every sampled frame of the second half one colour is a
+    hang. Anything else is unknown: a run is never called a boot because
+    its frames differ from another run's.
+    """
+    ended = rep.get("exit")
+    if ended and ended.get("timeout_s") is not None:
+        return HANG, {"rule": "timeout", "detail": f"killed after {ended['timeout_s']} s",
+                      "exit_status": ended.get("status"), "signal": ended.get("signal")}
+    if ended:
+        how = f"signal {ended['signal']}" if ended.get("signal") else f"exit status {ended.get('status')}"
+        return CRASH, {"rule": "exit_status", "detail": f"the worker died: {how}",
+                       "exit_status": ended.get("status"), "signal": ended.get("signal")}
+    frames = sampled(rep)
+    final = (rep.get("observables") or {}).get("frame_sha1")
+    if final:
+        frames.append((int(rep.get("frames", 0)) + int(rep.get("warmup", 0)), final))
+    references = {r["sha1"]: r for r in oracle.get("references") or ()}
+    reached = [(frame, digest) for frame, digest in frames if digest in references]
+    stages = list(dict.fromkeys(references[d]["stage"] for _, d in reached))
+    # A core that hides the BIOS screens by default still says it loaded one.
+    if "bios" not in stages and any(re.search(pattern, line["text"])
+                                    for pattern in oracle.get("bios_patterns") or ()
+                                    for line in rep.get("logs") or ()):
+        stages.insert(0, "bios")
+    game = [(frame, digest) for frame, digest in reached if references[digest]["stage"] == "game"]
+    if game:
+        frame, digest = game[0]
+        return BOOTS, {"rule": "reference_frame",
+                       "detail": f"frame {frame} shows {references[digest]['shows']}",
+                       "frame": frame, "sha1": digest, "stages": stages}
+    verdict, evidence = _without_the_game(rep, oracle, frames)
+    if stages:
+        # The console's own screens without one of the game: never a boot.
+        evidence["stages"] = stages
+        evidence["detail"] += ", after the BIOS screens and no screen of the game"
+    return verdict, evidence
+
+
+def _without_the_game(rep: dict, oracle: dict, frames: list[tuple[int, str]]) -> tuple[str, dict]:
+    """The verdict of a run that reached no screen of the game."""
+    for pattern in oracle.get("error_patterns") or ():
+        for line in rep.get("logs") or ():
+            if re.search(pattern, line["text"]):
+                return BIOS_ERROR, {"rule": "error_pattern",
+                                    "detail": f"the core logged an error at frame {line['frame']}",
+                                    "pattern": pattern, "log": line["text"], "frame": line["frame"]}
+    screens = {s["sha1"]: s for s in oracle.get("error_screens") or ()}
+    for frame, digest in frames:
+        if digest in screens:
+            return BIOS_ERROR, {"rule": "error_screen",
+                                "detail": f"frame {frame} shows {screens[digest]['shows']}",
+                                "frame": frame, "sha1": digest}
+    if not rep.get("loaded"):
+        return UNKNOWN, {"rule": "no_evidence",
+                         "detail": "retro_load_game returned false and no error pattern matched"}
+    uniform = set((rep.get("observables") or {}).get("uniform_frames") or ())
+    tail = [digest for frame, digest in sampled(rep)][len(sampled(rep)) // 2:]
+    if tail and all(digest in uniform for digest in tail):
+        return HANG, {"rule": "uniform_tail",
+                      "detail": f"the last {len(tail)} sampled frames are one colour each"}
+    if not oracle.get("references"):
+        return UNKNOWN, {"rule": "no_evidence", "detail": "no reference frame is known for this "
+                                                          "content on this core"}
+    return UNKNOWN, {"rule": "no_evidence", "detail": "no reference frame reached, no error pattern "
+                                                      "and no error screen"}
+
+
+def variant_verdict(reps: Sequence[dict]) -> tuple[str, dict]:
+    """The verdict all repetitions agree on, else unknown."""
+    verdicts = [rep["verdict"] for rep in reps]
+    if len(set(verdicts)) == 1:
+        first = reps[0]["verdict_evidence"]
+        evidence = {"rule": "repetitions_agree",
+                    "detail": f"{len(reps)} of {len(reps)} repetitions: {first['detail']}",
+                    "repetitions": verdicts}
+        for key in ("frame", "sha1", "stages", "pattern", "log", "exit_status", "signal"):
+            if key in first:
+                evidence[key] = first[key]
+        return verdicts[0], evidence
+    return UNKNOWN, {"rule": "repetitions_disagree",
+                     "detail": f"the repetitions disagree: {', '.join(verdicts)}",
+                     "repetitions": verdicts}
+
+
+def load_oracle(path: Path | None) -> tuple[dict, dict | None]:
+    """The oracle file's data and its identity, or nothing without a file."""
+    if path is None or not path.is_file():
+        return {}, None
+    import yaml  # noqa: PLC0415
+
+    data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    try:
+        shown = path.resolve().relative_to(ROOT).as_posix()
+    except ValueError:
+        shown = path.name
+    return data, {"file": shown, "sha256": file_digest(path, "sha256")}
+
+
 def default_emulator(core_file: Path, library_name: str | None) -> str:
     """The profile whose cores: names the .so, else the library name as a slug."""
     stem = core_file.name.split(".")[0].removesuffix("_libretro")
@@ -1143,14 +1441,21 @@ def probe(args: argparse.Namespace) -> int:
         for problem in problems:
             print(f"ERROR: {problem}", file=sys.stderr)
         return 2
+    with host_probe_lock():
+        return _probe(args)
+
+
+def _probe(args: argparse.Namespace) -> int:
     max_load = args.max_load if args.max_load is not None else (os.cpu_count() or 1) / 4
     load_before = load_average()
     if args.record and load_before and load_before[0] > max_load:
         print(f"ERROR: load average {load_before[0]} exceeds {max_load}: the machine is busy, "
               f"nothing recorded (raise --max-load to override)", file=sys.stderr)
         return 3
-    repetitions = 1 if args.mode == "run" else args.repeat
+    repetitions = args.repeat
     base = {dest: Path(src) for dest, src in args.system_file}
+    args.oracle_data, oracle_identity = load_oracle(Path(args.oracle) if args.oracle else DEFAULT_ORACLE)
+    args.content_sha1 = file_digest(Path(args.content)) if args.content else None
     work = Path(tempfile.mkdtemp(prefix=f"{args.mode}-", dir=_ensure(Path(args.work_dir))))
     try:
         variants = [run_variant(args, base, variant, work / f"v{i}", repetitions)
@@ -1188,6 +1493,9 @@ def probe(args: argparse.Namespace) -> int:
                            + (["frame_hashes", "uniform_frames"] if args.frame_hash_every else []),
             "frame_hash_every": args.frame_hash_every,
             "joypad_ports": args.joypad_ports,
+            "oracle": ({**oracle_identity, **oracle_for(args.oracle_data, args.content_sha1,
+                                                          first.get("library_name"))}
+                       if oracle_identity else None),
         },
         "variants": variants,
         "flags": [f"{v['label']}: {flag}" for v in variants for flag in v["flags"]],
@@ -1202,7 +1510,8 @@ def probe(args: argparse.Namespace) -> int:
     if args.mode == "check":
         record["check"] = {"claim": args.claim, "method": args.method, "observable": args.observable,
                            "variable": args.vary[0][0],
-                           "result": check_result(variants, args.observable)}
+                           "result": {**check_result(variants, args.observable),
+                                      "verdicts": {v["label"]: v["verdict"] for v in variants}}}
     try:
         record["emulator"] = args.emulator or default_emulator(Path(args.core), first.get("library_name"))
     except ProbeError as exc:

scripts/generate_site.py

@@ -7180,6 +7180,11 @@ def system_pages(
             _section_label(manufacturer, console),
             _install_pairs(installs.get(console, []), lead, consumers))
 
+    def region_block(console: str, level: int = 3) -> list[str]:
+        # Under the console's heading on a maker's page, a section of its
+        # own on the console's page.
+        return sitemeasurements.region_section(_REGION_RESULTS.get((slug, console), []), level)
+
     def all_sections() -> dict[str, list[Card]]:
         anchors: set[str] = set()
         return {console: cards_of(console, anchors) for console in ordered}
@@ -7221,6 +7226,7 @@ def system_pages(
             lines.append("")
             lines.extend(start_of(console))
             lines.extend(install_block(console))
+            lines.extend(region_block(console))
             lines.extend(_cards_markdown(cards))
             lines.append("")
         lines.extend(_next_block([*_NEXT_FROM_SYSTEMS, json_next]))
@@ -7283,6 +7289,7 @@ def system_pages(
         if console not in moved:
             summary.extend(start_of(console))
             summary.extend(install_block(console))
+            summary.extend(region_block(console))
             summary.extend(_cards_markdown(cards_of(console, summary_anchors)))
             summary.append("")
             continue
@@ -7367,6 +7374,8 @@ def system_pages(
             lines.extend(legend)
             lines.extend(_romset_note({console: consoles[console]}))
             lines.extend(install_block(console))
+            if index == 0:
+                lines.extend(region_block(console, 2))
             lines.extend(["## Files", ""])
             lines.extend(_cards_markdown(part))
             lines.append("")
@@ -9097,6 +9106,9 @@ _DATA_MEMBERS: dict[str, dict[str, dict]] = {}
 _DATA_REVISIONS: dict[str, dict] = {}
 # The measurement index items of each emulator, for its Tested block.
 _MEASUREMENT_ITEMS: dict[str, list[dict]] = {}
+# (manufacturer slug, console) -> the region outcomes of its systems
+# (region_check.py), shown under the console on its system page.
+_REGION_RESULTS: dict[tuple[str, str], list[dict]] = {}
 # The emulator pages that list each data directory, by ref.
 _DATA_READERS: dict[str, list[tuple[str, str]]] = {}
 
@@ -11833,6 +11845,13 @@ def _generate(args: argparse.Namespace) -> None:
         _system_pages_from_data(manufacturers, coverages, db, profiles)
     )
     print(f"  {len(_system_page_map)} system IDs mapped to pages")
+    from siterender import _page_by_norm  # noqa: PLC0415
+
+    _REGION_RESULTS.clear()
+    for result in sitemeasurements.site_region_results(repo=REPO_URL):
+        place = _system_page_map.get(result["system"]) or _page_by_norm(result["system"])
+        if place:
+            _REGION_RESULTS.setdefault(place, []).append(result)
     set_console_sections(manufacturers, program_names(profiles))
     from siterender import link_coverage, system_ids  # noqa: PLC0415
 
@@ -11887,7 +11906,9 @@ def _generate(args: argparse.Namespace) -> None:
     _MEASUREMENT_ITEMS.update(sitemeasurements.by_emulator(measured.index))
     print(f"  {measured.index['records']} records, {measured.index['count']} tests")
     write_if_changed(str(docs / sitemeasurements.EXPLORER_PAGE),
-                     sitemeasurements.generate_explorer_page(measured.index) + reference_next())
+                     sitemeasurements.generate_explorer_page(
+                         measured.index, {sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))})
+                     + reference_next())
 
     print("Generating static API and bulk data exports...")
     exports = generate_data_exports(

scripts/measurement_records.py

@@ -27,7 +27,9 @@ from collections.abc import Iterable
 from dataclasses import dataclass
 from pathlib import Path
 
-RECORD_VERSION = 3
+RECORD_VERSION = 4
+# 3: before the boot oracle. 4: every repetition and variant carries a verdict.
+RECORD_VERSIONS = (3, 4)
 RECORDS_DIR = "measurements"
 ID_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$")
 SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:[-_][a-z0-9]+)*$")
@@ -137,8 +139,8 @@ def check_line(record: dict) -> list[str]:
     """The structural rules a line holds, without a JSON Schema validator."""
     rid = record.get("id", "")
     problems = []
-    if record.get("record_version") != RECORD_VERSION:
-        problems.append(f"{rid}: record_version {record.get('record_version')} != {RECORD_VERSION}")
+    if record.get("record_version") not in RECORD_VERSIONS:
+        problems.append(f"{rid}: record_version {record.get('record_version')} not in {RECORD_VERSIONS}")
     if not ID_PATTERN.match(str(rid)):
         problems.append(f"{rid!r}: not an id of the form YYYY-MM-DD-<12 hex>")
     elif str(record.get("recorded_at", ""))[:10] != rid[:10]:

scripts/region_check.py

@@ -0,0 +1,400 @@
+#!/usr/bin/env python3
+"""Region checks: which regional BIOS a core reads for a game, and whether
+the game boots without it.
+
+For a system whose profile marks its BIOS entries with `region:`, each test
+game of a region (tests/campaign/content.yml) runs through core_probe four
+times, each a record in measurements/<emulator>.jsonl:
+
+  full    every regional BIOS the profile declares
+  own     only the entries whose `region:` holds the game's region
+  others  only the entries of the other regions
+  pack    the files of `generate_pack.py --emulator E --region R`, at
+          the place the pack puts them
+
+The outcome reads the records: the file the core opened in the full run
+(from strace) settles the `region:` claim, the verdicts of `own` and
+`others` say whether the BIOS is locked to the region, and `pack` says
+whether the pack filtered for the region boots the region's game.
+
+  run     the four runs for each region game of one core and system
+  report  the outcome of every region record already in measurements/
+
+A maintainer starts a run. Nothing here runs on a timer.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+import tempfile
+import time
+import urllib.request
+from collections.abc import Callable, Iterable
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import yaml  # noqa: E402
+
+ROOT = Path(__file__).resolve().parent.parent
+CONTENT_FILE = ROOT / "tests" / "campaign" / "content.yml"
+CORES = ROOT / ".cache" / "region" / "cores"
+NIGHTLY = "https://buildbot.libretro.com/nightly/linux/x86_64/latest/{core}_libretro.so.zip"
+VARIANTS = ("full", "own", "others", "pack")
+SUBJECT = re.compile(r"^region-(?P<core>[a-z0-9_]+)-(?P<game>[a-z0-9-]+)-(?P<variant>full|own|others|pack)$")
+# The territory a test game names, as generate_pack --region takes it.
+PACK_REGION = {"japan": "jp", "north-america": "us", "europe": "eu"}
+REGION_WORDS = {"japan": "Japanese", "north-america": "US", "europe": "European"}
+
+
+# --------------------------------------------------------------------------
+# Pure rules
+# --------------------------------------------------------------------------
+
+def regions_of(entry: dict) -> list[str]:
+    regions = entry.get("region") or []
+    return [regions] if isinstance(regions, str) else list(regions)
+
+
+def entry_dest(entry: dict) -> str:
+    path = entry.get("path")
+    if isinstance(path, str) and path and not path.endswith("/"):
+        return path
+    return entry["name"]
+
+
+def region_entries(profile: dict, system: str) -> list[dict]:
+    """The BIOS entries of a system that a `region:` ties to territories."""
+    out = []
+    for entry in profile.get("files") or ():
+        systems = entry.get("systems") or ([entry["system"]] if entry.get("system")
+                                            else profile.get("systems") or [])
+        if (system in systems and regions_of(entry) and "world" not in regions_of(entry)
+                and entry.get("category", "bios") == "bios" and entry.get("mode") != "standalone"):
+            out.append(entry)
+    return out
+
+
+def plan_variants(entries: Iterable[dict], region: str,
+                  local: Callable[[dict], Path | None]) -> dict[str, dict[str, Path]]:
+    """The staged files of `full`, `own` and `others`, dest -> file. An entry
+    the collection does not hold is left out of every variant."""
+    held = [(entry, local(entry)) for entry in entries]
+    held = [(entry, path) for entry, path in held if path]
+    full = {entry_dest(e): p for e, p in held}
+    own = {entry_dest(e): p for e, p in held if region in regions_of(e)}
+    others = {entry_dest(e): p for e, p in held if region not in regions_of(e)}
+    return {"full": full, "own": own, "others": others}
+
+
+def subject(core: str, game: str, variant: str) -> str:
+    words = re.findall(r"[a-z0-9]+", game.lower())
+    return f"region-{core.lower().replace('-', '_')}-{'-'.join(words)[:60].strip('-')}-{variant}"
+
+
+def opened(record: dict) -> set[str]:
+    return {item["path"] for variant in record["variants"] for rep in variant["repetitions"]
+            for item in rep.get("files_opened") or ()}
+
+
+def frames(record: dict) -> list:
+    rep = record["variants"][0]["repetitions"][0]
+    return (rep.get("observables") or {}).get("frame_hashes") or []
+
+
+def verdict(record: dict) -> str:
+    return record["variants"][0].get("verdict") or "unknown"
+
+
+def outcome(records: dict[str, dict], entries: list[dict], region: str) -> dict:
+    """What the four records of one game on one core settle.
+
+    `selected` is the regional file the core opened with every one staged.
+    The `region:` claim is `confirmed` when every regional file it opened
+    holds the game's region, `contradicted` when none does, `ambiguous`
+    when it opened its own region's file and others too, `unread` when it
+    opened none. Each variant's result is its oracle verdict. While no
+    reference frame is known for the game, a variant that samples the same
+    frames as the full run (`as_full`) boots, and one that does not stays
+    without an answer: frames that differ never make a verdict.
+    """
+    by_dest = {entry_dest(e): e for e in entries}
+    full = records.get("full")
+    selected = sorted(opened(full) & set(by_dest)) if full else []
+    if not full:
+        claim = "unrun"
+    elif not selected:
+        claim = "unread"
+    elif all(region in regions_of(by_dest[d]) for d in selected):
+        claim = "confirmed"
+    elif any(region in regions_of(by_dest[d]) for d in selected):
+        # Opened every region's file, to hash them or to choose: the open
+        # alone does not say which one the core runs.
+        claim = "ambiguous"
+    else:
+        claim = "contradicted"
+    results: dict[str, dict] = {}
+    for label in VARIANTS:
+        record = records.get(label)
+        if not record:
+            continue
+        loaded = all(rep.get("loaded") for v in record["variants"] for rep in v["repetitions"])
+        item = {"record": record["id"], "verdict": verdict(record), "loaded": loaded,
+                "staged": sorted(record["variants"][0]["system_files"])}
+        if full and label != "full":
+            item["as_full"] = loaded and frames(record) == frames(full)
+        results[label] = item
+
+    full_verdict = verdict(full) if full else None
+
+    def boots(label: str) -> bool | None:
+        item = results.get(label)
+        if not item:
+            return None
+        if item["verdict"] != "unknown":
+            return item["verdict"] == "boots"
+        # No verdict: the interim rule. The same frames as the full run is a
+        # boot, while the full run itself is not shown to fail. Other frames
+        # say nothing: the US disc reaches its game on the Japanese and
+        # European BIOS through another licence screen.
+        if full_verdict in ("boots", "unknown") and item.get("as_full"):
+            return True
+        return None
+
+    return {
+        "region": region,
+        "selected": [by_dest[d]["name"] for d in selected],
+        "claim": claim,
+        "variants": results,
+        "own_boots": boots("own"),
+        "others_boot": boots("others"),
+        "pack_boots": boots("pack"),
+        "judged_by": "verdict" if full_verdict not in (None, "unknown") else "full_run_frames",
+    }
+
+
+def sentence(game_region: str, result: dict) -> str:
+    """The outcome in words, for the site."""
+    word = REGION_WORDS.get(game_region, game_region)
+    own, others = result.get("own_boots"), result.get("others_boot")
+    if own is None:
+        return f"The {word} game has no run with its own BIOS yet."
+    first = f"The {word} BIOS {'boots' if own else 'does not boot'} the {word} game"
+    if others is None:
+        return first + ". The other regions' BIOS have no verdict yet."
+    if others:
+        return first + ", and so do the other regions' BIOS."
+    return first + ", the other regions' BIOS do not."
+
+
+# --------------------------------------------------------------------------
+# Runs
+# --------------------------------------------------------------------------
+
+def load_content(path: Path = CONTENT_FILE) -> dict[str, list[dict]]:
+    data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    out: dict[str, list[dict]] = {}
+    for system, spec in (data.get("systems") or {}).items():
+        for item in spec.get("content") or ():
+            out.setdefault(system, []).append({**item, "romfetch": spec["romfetch"]})
+    return out
+
+
+def content_file(item: dict, content_dir: Path) -> Path:
+    folder = content_dir / item["romfetch"] / item["name"]
+    if item.get("load") == "cue":
+        cues = sorted(folder.glob("*.cue"))
+        if len(cues) != 1:
+            raise FileNotFoundError(f"{folder}: {len(cues)} .cue files, one expected")
+        return cues[0]
+    if item.get("load"):
+        return folder / item["load"]
+    files = [p for p in sorted(folder.iterdir()) if p.is_file() and p.name != "identity.json"]
+    if len(files) != 1:
+        raise FileNotFoundError(f"{folder}: {len(files)} files, name one with load:")
+    return files[0]
+
+
+def nightly_core(core: str) -> tuple[Path, str]:
+    """The buildbot nightly of a core, downloaded once, and its URL."""
+    import zipfile  # noqa: PLC0415
+
+    url = NIGHTLY.format(core=core)
+    target = CORES / f"{core}_libretro.so"
+    if not target.is_file():
+        CORES.mkdir(parents=True, exist_ok=True)
+        archive = CORES / f"{core}.zip"
+        with urllib.request.urlopen(url, timeout=300) as response, archive.open("wb") as out:
+            out.write(response.read())
+        with zipfile.ZipFile(archive) as handle:
+            member = f"{core}_libretro.so"
+            if member not in handle.namelist():
+                raise FileNotFoundError(f"{archive.name}: no {member} inside")
+            partial = target.with_suffix(".part")
+            partial.write_bytes(handle.read(member))
+            partial.replace(target)
+        archive.unlink()
+    return target, url
+
+
+def pack_files(emulator: str, region: str, work: Path) -> dict[str, Path]:
+    """The files generate_pack puts in the emulator's pack for one region,
+    extracted, by their path in the pack."""
+    from ziptools import safe_extract_zip  # noqa: PLC0415
+
+    out_dir = work / "pack"
+    subprocess.run([sys.executable, str(ROOT / "scripts" / "generate_pack.py"), "--emulator", emulator,
+                    "--region", PACK_REGION[region], "--output-dir", str(out_dir)],
+                   cwd=ROOT, check=True, capture_output=True, text=True)
+    zips = sorted(out_dir.glob("*.zip"))
+    if len(zips) != 1:
+        raise FileNotFoundError(f"{out_dir}: {len(zips)} packs, one expected")
+    tree = work / "pack-tree"
+    safe_extract_zip(str(zips[0]), str(tree))
+    return {p.relative_to(tree).as_posix(): p for p in sorted(tree.rglob("*")) if p.is_file()
+            and not p.name.upper().startswith("README")}
+
+
+def local_resolver() -> Callable[[dict], Path | None]:
+    from common import load_database, resolve_local_file  # noqa: PLC0415
+
+    db = load_database(str(ROOT / "database.json"))
+
+    def local(entry: dict) -> Path | None:
+        hint = entry.get("path") if isinstance(entry.get("path"), str) else ""
+        path, status = resolve_local_file(entry, db, dest_hint=hint or "")
+        if not path or status in ("not_found", "hash_mismatch"):
+            return None
+        found = Path(path) if Path(path).is_absolute() else ROOT / path
+        return found if found.is_file() else None
+
+    return local
+
+
+BUSY = 3  # core_probe's exit status when it refuses to record on a busy host
+POLL_S = 15
+
+
+def probe_when_idle(argv: list[str], max_wait: float,
+                    clock: Callable[[], float] = time.monotonic,
+                    pause: Callable[[float], None] = time.sleep,
+                    runner: Callable[..., subprocess.CompletedProcess] = subprocess.run
+                    ) -> subprocess.CompletedProcess:
+    """core_probe, run again while it refuses a busy host, as the campaign
+    does: every POLL_S seconds until max_wait seconds have passed. The load
+    threshold stays core_probe's own."""
+    started = clock()
+    while True:
+        proc = runner(argv, cwd=ROOT, capture_output=True, text=True)
+        if proc.returncode != BUSY or clock() - started >= max_wait:
+            return proc
+        pause(POLL_S)
+
+
+def run(args: argparse.Namespace) -> int:
+    from common import load_emulator_profiles  # noqa: PLC0415
+
+    profile = load_emulator_profiles(str(ROOT / "emulators"))[args.emulator]
+    entries = region_entries(profile, args.system)
+    if not entries:
+        print(f"ERROR: {args.emulator} declares no regional BIOS for {args.system}", file=sys.stderr)
+        return 2
+    games = [g for g in load_content().get(args.system, []) if g.get("region") in PACK_REGION]
+    if not games:
+        print(f"ERROR: content.yml has no regional game for {args.system}", file=sys.stderr)
+        return 2
+    core, url = (Path(args.core), args.core_artifact) if args.core else nightly_core(args.core_name)
+    local = local_resolver()
+    failures = 0
+    for game in games:
+        load = content_file(game, Path(args.content_dir))
+        with tempfile.TemporaryDirectory(dir=ROOT / "tmp") as scratch:
+            variants = plan_variants(entries, game["region"], local)
+            variants["pack"] = pack_files(args.emulator, game["region"], Path(scratch))
+            for label in args.variants or VARIANTS:
+                argv = [sys.executable, str(ROOT / "scripts" / "core_probe.py"), "run",
+                        "--core", str(core), "--content", str(load), "--content-label", game["name"],
+                        "--frames", str(game.get("frames", 3600)),
+                        "--frame-hash-every", str(game.get("frame_hash_every", 10)),
+                        "--record", str(ROOT), "--emulator", args.emulator,
+                        "--subject", subject(core.name.removesuffix("_libretro.so"), game["name"], label),
+                        "--require-sandbox", "--timeout", str(args.timeout)]
+                if url:
+                    argv += ["--core-artifact", url]
+                for dest, src in sorted(variants[label].items()):
+                    argv += ["--system-file", f"{dest}={src}"]
+                proc = probe_when_idle(argv, args.max_load_wait)
+                status = "recorded" if proc.returncode == 0 else f"exit {proc.returncode}"
+                print(json.dumps({"game": game["name"], "variant": label, "status": status,
+                                  "error": proc.stderr.strip()[-400:] if proc.returncode else None}),
+                      flush=True)
+                failures += proc.returncode != 0
+    return 1 if failures else 0
+
+
+def collect(root: Path = ROOT, stored_records: list | None = None) -> list[dict]:
+    """Every region outcome the records under measurements/ hold. A game
+    listed under several systems (sega-megacd and mega-cd) is judged under
+    the one whose profile ran it."""
+    from common import load_emulator_profiles  # noqa: PLC0415
+    from measurement_records import live, read_records  # noqa: PLC0415
+
+    content: dict[str, list[tuple[str, dict]]] = {}
+    for system, items in load_content(root / "tests" / "campaign" / "content.yml").items():
+        for item in items:
+            content.setdefault(item["name"], []).append((system, item))
+    profiles = load_emulator_profiles(str(root / "emulators"))
+    groups: dict[tuple[str, str, str], dict[str, dict]] = {}
+    for stored in read_records(root) if stored_records is None else stored_records:
+        match = SUBJECT.match(stored.record.get("subject", ""))
+        label = ((stored.record.get("inputs") or {}).get("content") or {}).get("label")
+        if not match or label not in content:
+            continue
+        key = (stored.emulator, match["core"], label)
+        groups.setdefault(key, {})[match["variant"]] = stored.record
+    out = []
+    for (emulator, core, game), records in sorted(groups.items()):
+        records = {k: v for k, v in records.items() if v["id"] in {r["id"] for r in live(records.values())}}
+        profile = profiles.get(emulator, {})
+        system, item = next(((s, i) for s, i in content[game] if s in (profile.get("systems") or ())),
+                            content[game][0])
+        entries = region_entries(profile, system)
+        result = outcome(records, entries, item["region"])
+        out.append({"emulator": emulator, "core": core, "system": system, "game": game, **result,
+                    "sentence": sentence(item["region"], result)})
+    return out
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = parser.add_subparsers(dest="command", required=True)
+    go = sub.add_parser("run", allow_abbrev=False, help="the four runs of each region game, recorded")
+    go.add_argument("--emulator", required=True, help="the profile, e.g. beetle_psx")
+    go.add_argument("--system", required=True, help="the system, e.g. sony-playstation")
+    go.add_argument("--core-name", help="buildbot core name, e.g. mednafen_psx (downloads the nightly)")
+    go.add_argument("--core", help="a core .so to use instead of the nightly")
+    go.add_argument("--core-artifact", help="where --core was downloaded from")
+    go.add_argument("--content-dir", default=str(ROOT / ".cache" / "test-content"),
+                    help="where romfetch keeps the test games (default: .cache/test-content)")
+    go.add_argument("--variants", nargs="*", choices=VARIANTS, help="a subset of the four runs")
+    go.add_argument("--timeout", type=int, default=900, help="seconds per run")
+    go.add_argument("--max-load-wait", type=int, default=1800,
+                    help="seconds a run waits for a busy host to settle before it is given up (default 1800)")
+    sub.add_parser("report", allow_abbrev=False, help="the outcomes the records hold, as JSON")
+    args = parser.parse_args(argv)
+    if args.command == "run":
+        if not (args.core or args.core_name):
+            parser.error("run needs --core-name or --core")
+        return run(args)
+    print(json.dumps(collect(), indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    from common import run_cli
+
+    run_cli(main)

scripts/sitemeasurements.py

@@ -18,17 +18,16 @@ its line at the last commit that touched measurements/.
 
 from __future__ import annotations
 
+import html
 import json
 import os
 import re
 import sqlite3
 import sys
-from collections.abc import Callable
 from dataclasses import dataclass
 from pathlib import Path
 
 import measurement_records as mr
-import siteentries
 import sitemarkup as markup
 from artifacts import json_text, write_if_changed
 from siterender import _escape_tags
@@ -41,7 +40,7 @@ HOSTS_MARKER = "<!-- measurements-hosts -->"
 INDEX_PATH = Path("data") / "measurements" / "index.json"
 SQLITE_PATH = Path("downloads") / "measurements.sqlite"
 PER_EMULATOR_DIR = Path("downloads") / "measurements"
-SQLITE_SCHEMA_VERSION = "2"
+SQLITE_SCHEMA_VERSION = "3"
 # Observables the variants table leaves out: one value per sampled frame.
 BULKY_OBSERVABLES = ("frame_hashes",)
 INDEX_SCHEMA_VERSION = 1
@@ -76,6 +75,40 @@ def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str
     return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
 
 
+def verdicts_of(record: dict) -> list[dict]:
+    """Each variant's boot verdict, from record version 4 on."""
+    return [{"variant": v["label"], "verdict": v["verdict"],
+             "detail": (v.get("verdict_evidence") or {}).get("detail", "")}
+            for v in record["variants"] if "verdict" in v]
+
+
+def result_of(record: dict) -> tuple[str, str]:
+    """The status and statement a page shows for a record.
+
+    A run, or a check compared on the sampled frames, is shown by its boot
+    verdicts: one verdict when every variant has it, else `mixed`, each
+    variant named in the statement. A check on another observable keeps
+    its comparison. A frame comparison recorded before the boot oracle
+    shows `no verdict`: frames that differ prove nothing about a boot.
+    """
+    check = record.get("check") or {}
+    result = check.get("result") or {}
+    if record["kind"] == "bench":
+        return "measured", bench_statement(record) if record.get("timing") else ""
+    verdicts = verdicts_of(record)
+    by_frames = record["kind"] == "run" or check.get("observable") == "frame_hashes"
+    if verdicts and by_frames:
+        values = {v["verdict"] for v in verdicts}
+        status = values.pop() if len(values) == 1 else "mixed"
+        if record["kind"] == "run":
+            return status, verdicts[0]["detail"]
+        return status, ". ".join(f"{v['variant']}: {v['verdict']}" for v in verdicts)
+    if by_frames:
+        return "no verdict", ("recorded before the boot oracle. Its frames were compared, which "
+                              "settles nothing about a boot")
+    return result.get("status") or "ran", result.get("statement") or ""
+
+
 def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
                system_names: dict[str, str]) -> dict:
     record = entry.record
@@ -95,9 +128,9 @@ def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
         "method": check.get("method"),
         "observable": check.get("observable"),
         "variable": check.get("variable"),
-        "status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
-        "statement": (check.get("result") or {}).get("statement")
-                     or (bench_statement(record) if record.get("timing") else ""),
+        "status": result_of(record)[0],
+        "statement": result_of(record)[1],
+        "verdicts": verdicts_of(record),
         "content": content.get("label"),
         "variants": len(record["variants"]),
         "repetitions": record["inputs"]["repetitions"],
@@ -114,9 +147,21 @@ def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
         "runs": runs,
         "record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line),
         "sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
+        "host_short": short_host(record["host"]),
+        "setup": setup_of(record),
+        "groups": variant_groups(record),
+        "proves": proves(record["id"], profile),
+        "timing": ({"ms_per_frame": record["timing"]["ms_per_frame"]["median"],
+                    "fps": record["timing"]["fps"]["median"],
+                    "stdev": record["timing"]["ms_per_frame"]["stdev"]}
+                   if record.get("timing") else None),
     }
 
 
+def with_title(item: dict) -> dict:
+    return {**item, "title": short_title(item)}
+
+
 def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
                 system_names: dict[str, str] | None = None) -> dict:
     """One item per (emulator, subject): its latest record not superseded."""
@@ -129,7 +174,8 @@ def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
     for key in sorted(by_subject):
         entries = by_subject[key]
         latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
-        items.append(index_item(latest, len(entries), source or {}, profiles, system_names or {}))
+        items.append(with_title(index_item(latest, len(entries), source or {}, profiles,
+                                           system_names or {})))
     items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
     return {
         "schema_version": INDEX_SCHEMA_VERSION,
@@ -206,7 +252,8 @@ def campaign_section(summary: dict | None) -> list[str]:
     lines = [
         "## Campaign",
         "",
-        f"Tests run by themselves every night on an idle computer ([how]({METHOD_PAGE})). "
+        f"A maintainer starts a pass, on an idle computer, and nothing runs by itself "
+        f"([how]({METHOD_PAGE})). "
         f"Last pass: {_plain(last['ended_at'][:10])}, {_plain(last['result'])}"
         + (f" ({_plain(last['detail'])})" if last.get("detail") else "") + ". "
         f"{int(counts['done'])} tests done, {int(counts['failed'])} failed, {int(counts['pending'])} waiting. "
@@ -241,6 +288,176 @@ def campaign_section(summary: dict | None) -> list[str]:
     return lines
 
 
+# --------------------------------------------------------------------------
+# Coverage and region results
+# --------------------------------------------------------------------------
+
+CONTENT_FILE = Path("tests") / "campaign" / "content.yml"
+
+
+def coverage(root: Path, stored: list[mr.Stored], profiles: dict | None = None) -> dict:
+    """Per system of the content list: its games, and which emulators ran
+    each one. A record counts for a system only when its emulator's profile
+    names that system: one catalog name can be a PlayStation disc and a
+    Saturn disc. A system no catalog covers comes with the reason and the
+    error romfetch returned."""
+    import yaml  # noqa: PLC0415
+
+    path = root / CONTENT_FILE
+    if not path.is_file():
+        return {"systems": [], "uncovered": []}
+    data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+    ran: dict[str, set[str]] = {}
+    for entry in stored:
+        label = ((entry.record.get("inputs") or {}).get("content") or {}).get("label")
+        if label:
+            ran.setdefault(label, set()).add(entry.emulator)
+    systems = []
+    for system, spec in (data.get("systems") or {}).items():
+        games = [{"name": item["name"], "region": item.get("region"),
+                  "emulators": sorted(e for e in ran.get(item["name"], ())
+                                      if profiles is None
+                                      or system in ((profiles.get(e) or {}).get("systems") or ()))}
+                 for item in spec.get("content") or ()]
+        listed = {item["name"] for item in spec.get("content") or ()}
+        # Runs of this system's emulators on games the list does not name.
+        other = sorted({e for label, emulators in ran.items() if label not in listed for e in emulators
+                        if profiles is not None
+                        and system in ((profiles.get(e) or {}).get("systems") or ())})
+        systems.append({"system": system, "games": games, "other_games_by": other,
+                        "tested": any(game["emulators"] for game in games)})
+    uncovered = [{"system": system, "want": item.get("want"), "reason": " ".join(str(item["reason"]).split()),
+                  "romfetch": item.get("romfetch") or {}}
+                 for system, item in (data.get("uncovered") or {}).items()]
+    return {"systems": systems, "uncovered": uncovered}
+
+
+def coverage_section(cov: dict | None, system_names: dict[str, str] | None = None) -> list[str]:
+    """The Coverage block of the explorer page: tested or not, and why not."""
+    if not cov or not (cov.get("systems") or cov.get("uncovered")):
+        return []
+    names = system_names or {}
+    tested = [s for s in cov["systems"] if s["tested"]]
+    lines = [
+        "## Coverage",
+        "",
+        f"Test games come from [`{CONTENT_FILE.as_posix()}`]({REPO}/blob/main/{CONTENT_FILE.as_posix()}), "
+        f"one per region where a BIOS depends on it. {len(tested)} of {len(cov['systems'])} listed "
+        f"systems have a recorded run, and {len(cov['uncovered'])} "
+        f"{'system has' if len(cov['uncovered']) == 1 else 'systems have'} no test game.",
+        "",
+        "| System | Games | Tested by |",
+        "|--------|-------|-----------|",
+    ]
+    # Two ids of one console (sega-megacd, mega-cd) share their list: one row.
+    rows: dict[tuple[str, str], set[str]] = {}
+    others: dict[tuple[str, str], set[str]] = {}
+    for item in cov["systems"]:
+        row = (_plain(names.get(item["system"], item["system"])),
+               ", ".join(_plain(game["name"]) for game in item["games"]))
+        rows.setdefault(row, set()).update(e for game in item["games"] for e in game["emulators"])
+        others.setdefault(row, set()).update(item.get("other_games_by") or ())
+    for (name, games), by in rows.items():
+        if by:
+            status = ", ".join(_plain(e) for e in sorted(by))
+        elif others[(name, games)]:
+            status = ("not yet with these games, other games by "
+                      + ", ".join(_plain(e) for e in sorted(others[(name, games)])))
+        else:
+            status = "not yet: no run recorded"
+        lines.append(f"| {name} | {games} | {status} |")
+    lines.append("")
+    if cov["uncovered"]:
+        lines += ["Systems without a test game:", "",
+                  "| System | Wanted | Why not | romfetch |",
+                  "|--------|--------|---------|----------|"]
+        for item in cov["uncovered"]:
+            code = item["romfetch"].get("code") or "no source"
+            lines.append(f"| {_plain(names.get(item['system'], item['system']))} | {_plain(item['want'])} | "
+                         f"{_plain(item['reason'])} | exit {_plain(item['romfetch'].get('exit'))}, "
+                         f"{_code(code)} |")
+        lines.append("")
+    return lines
+
+
+def region_results(root: Path, stored: list[mr.Stored], source: dict | None) -> list[dict]:
+    """The region outcomes the records hold (region_check.py), each record
+    linked at the commit the explorer links."""
+    import region_check  # noqa: PLC0415
+
+    lines = {s.record["id"]: s for s in stored}
+    results = region_check.collect(root, stored)
+    for result in results:
+        for item in result["variants"].values():
+            found = lines.get(item["record"])
+            item["url"] = (permalink(source["repo"], source["commit"], mr.records_path(found.emulator),
+                                     found.line)
+                           if found and source and source.get("commit") else None)
+    return results
+
+
+def site_region_results(root: Path = ROOT, repo: str = REPO) -> list[dict]:
+    """region_results with the records and commit the explorer uses."""
+    stored = mr.read_records(root)
+    source = {"repo": repo, "commit": records_commit(root) if stored else None}
+    return region_results(root, stored, source)
+
+
+REGION_SUBJECT = re.compile(r"^region-[a-z0-9_]+-[a-z0-9-]+-(full|own|others|pack)$")
+
+VARIANT_WORDS = {"full": "every regional BIOS", "own": "its own region's BIOS only",
+                 "others": "the other regions' BIOS only", "pack": "the region's pack"}
+
+
+REGION_SHORT = {"japan": "Japanese", "north-america": "US", "europe": "European"}
+OUTCOME_WORDS = {True: "boots", False: "does not boot", None: "no verdict"}
+
+
+def core_region_line(results: list[dict]) -> str:
+    """One core's outcome over its region games, in a sentence: each game
+    on its own BIOS, then on the other regions' BIOS."""
+    own = {r["own_boots"] for r in results}
+    others = {r["others_boot"] for r in results}
+    if own == {True} and others == {True}:
+        return "Each region's game boots on its own BIOS and on the other regions' BIOS."
+    if own == {True} and others == {False}:
+        return "Each region's game boots on its own BIOS only."
+    parts = [f"{REGION_SHORT.get(r['region'], r['region'])} game: {OUTCOME_WORDS[r['own_boots']]} "
+             f"on its BIOS, {OUTCOME_WORDS[r['others_boot']]} on the others" for r in results]
+    return ", ".join(parts) + "."
+
+
+def region_section(results: list[dict], level: int = 3) -> list[str]:
+    """What the region runs showed for one system: a line per core, the
+    games and their records in a fold."""
+    if not results:
+        return []
+    lines = ["#" * level + " Region check", "",
+             f"Each game ran with {VARIANT_WORDS['own']}, then with {VARIANT_WORDS['others']} "
+             f"([method](../{METHOD_PAGE})).", ""]
+    by_core: dict[str, list[dict]] = {}
+    for result in results:
+        by_core.setdefault(result["core"], []).append(result)
+    for core, items in by_core.items():
+        lines.append(f"- **{_plain(core)}**: {_plain(core_region_line(items))}")
+    lines.append("")
+    rows = []
+    for result in results:
+        links = ", ".join(_link(item.get("url"), _cell(VARIANT_WORDS[label]))
+                          for label, item in result["variants"].items())
+        names = ", ".join(f"<code>{_cell(name)}</code>" for name in result["selected"])
+        read = (f" It reads {names} with every regional BIOS there "
+                f"({_cell(result['claim'])} against the profile)." if result["selected"] else "")
+        basis = ("" if result["judged_by"] == "verdict"
+                 else " Judged against the run with every regional BIOS until reference frames are recorded.")
+        rows.append(f"<li><strong>{_cell(result['core'])}</strong>, {_cell(result['game'])}: "
+                    f"{_cell(result['sentence'])}{read}{basis} Records: {links}.</li>")
+    # One line: MDX reads an HTML block split over lines as paragraphs.
+    lines += ['<details class="rb-region-runs"><summary>Each game and its records</summary><ul>'
+              + "".join(rows) + "</ul></details>", ""]
+    return lines
+
+
 # --------------------------------------------------------------------------
 # SQLite
 # --------------------------------------------------------------------------
@@ -294,7 +511,8 @@ SCHEMA_SQL = """
         run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
         label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
         flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
-        files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
+        files_probed_absent TEXT, tracing TEXT, verdict TEXT, verdict_evidence TEXT,
+        PRIMARY KEY (run_id, position)
     );
     CREATE TABLE system_files (
         run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
@@ -405,11 +623,12 @@ def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
         # The first repetition stands for the variant, its per-frame hashes left out.
         observables = {key: value for key, value in (first.get("observables") or {}).items()
                        if key not in BULKY_OBSERVABLES}
-        connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
+        connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
             rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)),
             json_text(variant["flags"]), json_text(observables),
             json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")),
-            json_text(first.get("tracing"))))
+            json_text(first.get("tracing")), variant.get("verdict"),
+            json_text(variant.get("verdict_evidence")) if "verdict_evidence" in variant else None))
         connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
             (rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
     for metric, stats in sorted((r.get("timing") or {}).items()):
@@ -462,6 +681,7 @@ def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulato
     profiles = load_profiles(emulators_dir)
     index = build_index(stored, source, profiles, system_names)
     index["campaign"] = campaign_summary(root, stored, source)
+    index["coverage"] = coverage(root, stored, profiles)
     paths = [public / INDEX_PATH]
     (public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
     write_if_changed(str(public / INDEX_PATH),
@@ -480,9 +700,14 @@ def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulato
 # --------------------------------------------------------------------------
 
 def by_emulator(index: dict) -> dict[str, list[dict]]:
+    """Each emulator's items, with the count of its findings still open."""
+    opened: dict[str, int] = {}
+    for line in (index.get("campaign") or {}).get("open") or ():
+        opened[line["emulator"]] = opened.get(line["emulator"], 0) + 1
     out: dict[str, list[dict]] = {}
     for item in index.get("items", []):
-        out.setdefault(item["emulator"], []).append(item)
+        out.setdefault(item["emulator"], []).append(
+            {**item, "open_findings": opened.get(item["emulator"], 0)})
     return out
 
 
@@ -498,92 +723,418 @@ def _short(commit: str | None) -> str:
     return f"`{commit[:12]}`" if commit else "unknown"
 
 
-# A result the probe could not settle is the one a reader should not take
-# at its word. The others describe what was seen, none of them is a fault.
-_STATUS_BADGE = {"inconclusive": "warning"}
+# A verdict or a comparison, as the block says it: words and a chip kind.
+RESULT_WORDS = {
+    "boots": ("works", "success"),
+    "bios_error": ("BIOS error", "danger"),
+    "crash": ("crash", "danger"),
+    "hang": ("hang", "warning"),
+    "unknown": ("unknown", "muted"),
+    "mixed": ("depends on the file", "info"),
+    "same": ("same with every file", "muted"),
+    "differs": ("depends on the file", "info"),
+    "inconclusive": ("inconclusive", "warning"),
+    "no verdict": ("no verdict", "muted"),
+    "ran": ("ran", "muted"),
+}
+# Words a subject slug spells that read as one term.
+TERMS = {"bios": "BIOS", "vcd": "Video CD", "cd": "CD", "jp": "JP", "us": "US", "eu": "EU",
+         "cr2": "CR2", "mpeg": "MPEG", "psx": "PSX", "hle": "HLE", "usa": "USA"}
+# The campaign's own check names, as the block says them.
+CHECK_NAMES = {"files": "full set", "none": "no BIOS", "negative": "wrong files",
+               "references": "reference run", "variants": "copies", "alias": "other name",
+               "path": "other path", "bench": "benchmark", "boot": "boot test", "alone": "file alone"}
 
 
-def _computer(host: str) -> str:
-    """The computer a host line names. The load average differs per run."""
-    return host.split(", load ")[0]
+def _words(text: str) -> list[str]:
+    return re.findall(r"[a-z0-9]+", str(text or "").lower())
 
 
-def _build(core: dict) -> str:
-    return f"{_text(core['library'])} at {_short(core['commit'])}" if core.get("commit") \
-        else _text(core["library"])
+def short_content(label: str | None) -> str:
+    """A content label cut to the game's name: no source prefix, no
+    edition, region or identity detail."""
+    text = str(label or "").strip()
+    if ":" in text.split("(")[0]:
+        text = text.split(":", 1)[1].strip()
+    for cut in (" - ", " (", ", "):
+        text = text.split(cut, 1)[0]
+    return text.strip()
 
 
-def _shared(items: list[dict], value: Callable[[dict], str]) -> str | None:
-    """The value every item gives, None when they differ."""
-    values = {value(item) for item in items}
-    return values.pop() if len(values) == 1 else None
+def short_title(item: dict) -> str:
+    """A test's name in a few words: the file it varies, the game, and what
+    the subject adds to them (a build, a region, a check of the campaign)."""
+    thing = Path(str(item.get("variable") or "")).name
+    game = short_content(item.get("content"))
+    known = set(_words(thing)) | set(_words(item.get("content"))) | set(_words(game))
+    known |= set(_words(Path(str((item.get("core") or {}).get("file") or "")).name.split("_libretro")[0]))
+    axis = []
+    for word in _words(item.get("subject")):
+        if word in known or re.fullmatch(r"[0-9a-f]{8}", word):
+            continue
+        issue = re.fullmatch(r"issue(\d+)", word)
+        word = f"issue {issue[1]}" if issue else CHECK_NAMES.get(word, TERMS.get(word, word))
+        if word not in axis:
+            axis.append(word)
+    head = ", ".join(part for part in (thing, game) if part)
+    title = f"{head} ({' '.join(axis)})" if axis and head else head or " ".join(axis)
+    return title or str(item.get("subject") or "")
 
 
-def _plural(count: int, noun: str) -> str:
-    return f"{count} {noun}{'s' if count != 1 else ''}"
+OBSERVABLE_WORDS = {"video_sha1": "video", "frame_hashes": "screens", "audio_sha1": "audio",
+                    "state_sha1": "save state", "frame_sha1": "last frame"}
 
 
-def tested_row(item: dict, shared_build: bool) -> siteentries.Row:
-    """One test: its claim as the row, the result beside it, the method,
-    the load and the record when the row opens."""
-    title = item["claim"] or f"Benchmark of {item['content'] or 'the core alone'}"
-    runs = "1 run" if item["runs"] == 1 else f"{item['runs']} runs, the latest shown"
-    body = []
-    if item["statement"]:
-        body += [f"{_text(item['statement'])}.", ""]
-    if item.get("method"):
-        body += [f"**Method:** {_text(item['method'])}", ""]
-    load = item["host"].partition(", load ")[2]
-    counts = (f"{_plural(item['variants'], 'variant')}, "
-              f"{_plural(item['repetitions'], 'repetition')} each"
-              + (f", {_plural(item['flags'], 'flag')} raised" if item["flags"] else "") + ".")
-    body += [f"**Ran:** {counts}" + (f" Load {_text(load)}." if load else "")
-             + ("" if shared_build else f" Core build: {_build(item['core'])}."), ""]
-    body += [f"[Full record]({item['record_url']}), {runs}."]
-    tone = _STATUS_BADGE.get(item["status"], "muted")
-    result = f'<span class="rb-badge rb-badge-{tone}">{_text(item["status"])}</span>'
-    return siteentries.Row(_text(title), body, [_text(item["content"]), result])
+def setup_of(record: dict) -> dict:
+    """What told two runs of one test apart, each as the words a title
+    would use: the campaign check, the core build, the core options, the
+    clock, the number of files tried and what was compared."""
+    build = str(record["core"].get("build_command") or "")
+    patched = re.search(r"\b(patch|edit(?:ed|ing)?)\b", build, re.IGNORECASE)
+    named = re.search(r"\b([A-Z]+[0-9]+)\b", build) if patched else None
+    check = record.get("check") or {}
+    first = (_words(record["subject"]) or [""])[0]
+    options = {}
+    for key, value in sorted(record["inputs"].get("options", {}).items()):
+        options[key] = f"{value} region" if key.endswith("_region") else f"{key.rsplit('_', 1)[-1]} {value}"
+    observable = str(check.get("observable") or "")
+    return {
+        "check": CHECK_NAMES.get(first, ""),
+        "build": (f"{named[1]} patch" if named else "patched build") if patched else "published build",
+        "options": options,
+        "clock": "fixed clock" if "FAKETIME" in record["inputs"].get("env", {}) else "running clock",
+        "files": (f"{len(record['variants'])} file{'s' if len(record['variants']) != 1 else ''}"
+                  if record["kind"] == "check" else ""),
+        "observable": (observable.split(":", 1)[1] if observable.startswith("state:")
+                       else OBSERVABLE_WORDS.get(observable, observable)),
+    }
+
+
+def _features(item: dict) -> dict[str, str]:
+    """A test's setup as named features, the options one by one."""
+    setup = item.get("setup") or {}
+    out = {name: str(setup.get(name) or "") for name in ("build", "clock", "files", "observable")}
+    for key, value in (setup.get("options") or {}).items():
+        out[f"option:{key}"] = value
+    return out
+
+
+def distinct_titles(items: list[dict]) -> dict[str, str]:
+    """A title per item: its game and campaign check, then what sets it
+    apart from the other tests of the same file, game and core.
+
+    A test whose setup is the common one of its group is named by its game.
+    Another adds the features where it leaves the common setup (a patched
+    build, a running clock, another region). Two titles still alike take
+    every feature that varies in the group, then the subject's own words.
+    Nothing is cut.
+    """
+    out: dict[str, str] = {}
+    peers: dict[tuple, list[dict]] = {}
+    for item in items:
+        game = short_content(item.get("content")) or short_title(item)
+        peers.setdefault((item.get("variable") or "", game, item["core"].get("file")), []).append(item)
+    for (_, game, _), group in peers.items():
+        features = [_features(item) for item in group]
+        names = sorted({name for f in features for name in f},
+                       key=lambda n: (n.startswith("option:"), ["build", "clock", "files", "observable"].index(n)
+                                      if not n.startswith("option:") else 0, n))
+        common = {}
+        for name in names:
+            values = [f.get(name, "") for f in features]
+            ranked = sorted(set(values), key=lambda v: -values.count(v))
+            # A tie has no common value: every test names its own.
+            tied = len(ranked) > 1 and values.count(ranked[0]) == values.count(ranked[1])
+            common[name] = None if tied else ranked[0]
+        varying = [n for n in names if len({f.get(n, "") for f in features}) > 1]
+
+        def title(item: dict, own: dict[str, str], every: bool, varying: list[str] = varying,
+                  common: dict[str, str] = common, game: str = game) -> str:
+            words = [(item.get("setup") or {}).get("check", "")]
+            for name in varying:
+                value = own.get(name, "")
+                if every or value != common[name]:
+                    words.append(value or f"no {name.rsplit('_', 1)[-1]}")
+            return ", ".join([game, *[w for w in words if w]])
+
+        for item, own in zip(group, features):
+            out[item["id"]] = title(item, own, every=False)
+        for every in (True, None):
+            seen: dict[str, int] = {}
+            for item in group:
+                seen[out[item["id"]]] = seen.get(out[item["id"]], 0) + 1
+            for item, own in zip(group, features):
+                if seen[out[item["id"]]] > 1:
+                    if every:
+                        out[item["id"]] = title(item, own, every=True)
+                    else:
+                        axis = short_title({**item, "variable": None, "content": None})
+                        out[item["id"]] = f"{out[item['id']]}, {axis}"
+    return out
+
+
+def short_host(host: dict) -> str:
+    """The computer as a reader names it: the CPU model and the OS."""
+    cpu = str(host.get("cpu_model") or "unknown CPU")
+    cpu = re.sub(r"\(R\)|\(TM\)|\bCPU\b|@.*$|^\d+(?:st|nd|rd|th) Gen ", "", cpu)
+    cpu = " ".join(cpu.split())
+    system = str(host.get("os") or "")
+    name = "Linux" if "Linux" in system else system.split("-")[0] or "unknown OS"
+    if "microsoft" in system.lower():
+        name += " (WSL2)"
+    return f"{cpu}, {name}"
+
+
+def variant_groups(record: dict) -> list[dict]:
+    """One entry per variant: the result group it falls in, and whether it
+    staged no file. A run that carries verdicts is grouped by verdict, a
+    comparison by the value its observable took."""
+    check = record.get("check") or {}
+    position: dict[str, int] = {}
+    result: dict[str, str] = {}
+    if all("verdict" in v for v in record["variants"]) and (
+            record["kind"] == "run" or check.get("observable") == "frame_hashes"):
+        for variant in record["variants"]:
+            position.setdefault(variant["verdict"], len(position))
+            result[variant["label"]] = RESULT_WORDS[variant["verdict"]][0]
+        groups = {v["label"]: position[v["verdict"]] for v in record["variants"]}
+    else:
+        groups = {}
+        for number, group in enumerate((check.get("result") or {}).get("groups") or ()):
+            for label in group["variants"]:
+                groups[label] = number
+                result[label] = f"result {chr(ord('A') + number % 26)}"
+    varied = check.get("variable")
+    return [{"variant": v["label"], "group": groups.get(v["label"], 0),
+             "result": result.get(v["label"], ""),
+             "absent": bool(varied) and varied in v["system_files"] and v["system_files"][varied] is None}
+            for v in record["variants"]]
+
+
+def proves(record_id: str, profile: dict) -> list[str]:
+    """The profile fields this record is evidence for, as `field of file`."""
+    out = []
+    for entry in profile.get("files") or ():
+        for name, ids in sorted((entry.get("evidence") or {}).items()):
+            if record_id in ids:
+                out.append(f"{name} of {entry.get('name')}")
+    return out
+
+
+# An MDX page reads a brace as the start of an expression, in text and in
+# an attribute alike.
+_BRACES = str.maketrans({"{": "&#123;", "}": "&#125;"})
+
+
+def _attr(value: object) -> str:
+    return html.escape(" ".join(str(value or "").split()), quote=True).translate(_BRACES)
+
+
+def _cell(value: object) -> str:
+    """Record text placed in raw HTML: escaped as text, never read as
+    Markdown (a run's output is no markup), braces made inert for MDX. A
+    semicolon joining two clauses ends a sentence, as on the rest of the
+    site."""
+    text = " ".join(str(value or "").split())
+    text = re.sub(r";\s+(\w)", lambda m: ". " + m[1].upper(), text)
+    return html.escape(text, quote=True).translate(_BRACES)
+
+
+_SAFE_HREF = re.compile(r"^(?:https?://[^\s\"'<>]+|/[^\s\"'<>]*|\.{0,2}/?[\w.~-][^\s\"'<>:]*)$")
+
+
+def _href(url: object) -> str | None:
+    """A link target from a record: http(s) or relative only, else none."""
+    text = str(url or "")
+    return _attr(text) if text and _SAFE_HREF.match(text) and not text.lower().startswith(
+        ("javascript:", "data:", "vbscript:")) else None
+
+
+def _link(url: object, text: str) -> str:
+    target = _href(url)
+    return f'<a href="{target}">{text}</a>' if target else text
+
+
+def _dots(item: dict) -> str:
+    groups = item.get("groups") or []
+    names = ". ".join(f"{g['variant']}: {g['result']}" for g in groups)
+    dots = "".join(
+        f'<span class="rb-dot rb-dot-{g["group"] % 5}{" rb-dot-empty" if g["absent"] else ""}" '
+        f'title="{_attr(g["variant"] + ": " + g["result"])}"></span>' for g in groups)
+    return (f'<span class="rb-dots" role="img" aria-label="{_attr(f"{len(groups)} variants. {names}")}">'
+            f"{dots}</span>")
+
+
+def _chip(item: dict) -> str:
+    words, kind = RESULT_WORDS.get(item["status"], (item["status"], "muted"))
+    return f'<span class="rb-badge rb-badge-{kind}">{_cell(words)}</span>'
+
+
+def _sentence_html(label: str, value: object, markup_ready: bool = False) -> str:
+    """One line of the details, or nothing: an empty value is never printed."""
+    text = (str(value or "") if markup_ready else _cell(value)).strip().rstrip(".")
+    return f"<p><strong>{label}:</strong> {text}.</p>" if text else ""
+
+
+def _details(item: dict) -> list[str]:
+    core = item["core"]
+    build = _cell(core["library"]) + (f" at <code>{_cell(core['commit'][:12])}</code>"
+                                      if core.get("commit") else "")
+    reps = f"{item['variants']} variant{'s' if item['variants'] != 1 else ''}, " \
+           f"{item['repetitions']} repetition{'s' if item['repetitions'] != 1 else ''} each"
+    if item["flags"]:
+        reps += f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised"
+    runs = "" if item["runs"] == 1 else f", {item['runs']} runs, the latest shown"
+    lines = [
+        _sentence_html("Claim", item.get("claim")),
+        _sentence_html("Result", item.get("statement")),
+        _sentence_html("Method", item.get("method")),
+        _sentence_html("Proves", ", ".join(item.get("proves") or [])),
+        _sentence_html("Ran on", item.get("host_short") or item["host"]),
+        _sentence_html("Core build", build and f"{build}. {reps}", markup_ready=True),
+        f'<p>{_link(item["record_url"], "Full record")}{runs}.</p>',
+    ]
+    return [line for line in lines if line]
+
+
+def tested_counts(items: list[dict]) -> dict:
+    tests = [i for i in items if i["kind"] != "bench"]
+    return {"tests": len(items), "confirmed": sum(1 for i in items if i.get("proves")),
+            "open": max((i.get("open_findings", 0) for i in items), default=0),
+            "benchmarks": len(items) - len(tests),
+            "last": max((i["date"][:10] for i in items), default="")}
+
+
+def core_label(item: dict) -> str:
+    return Path(str(item["core"].get("file") or "")).name.split("_libretro")[0]
+
+
+def _row(item: dict, title: str, page_host: str, main_core: str) -> list[str]:
+    elsewhere = (item.get("host_short") or item["host"]) != page_host
+    title = _cell(title)
+    if core_label(item) != main_core:
+        title += f' <small>{_cell(core_label(item))}</small>'
+    return [
+        '<details class="rb-tested-row">',
+        ("<summary>"
+         f'<span class="rb-tested-test">{title}</span>'
+         f'<span class="rb-tested-result">{_chip(item)}</span>'
+         f'<span class="rb-tested-variants">{_dots(item)}</span>'
+         f'<span class="rb-tested-date">{item["date"][:10]}'
+         + (f' <small>{_attr(item.get("host_short"))}</small>' if elsewhere else "")
+         + "</span></summary>"),
+        '<div class="rb-tested-more">',
+        *_details(item),
+        "</div>",
+        "</details>",
+    ]
+
+
+def by_file(items: list[dict]) -> list[tuple[str, list[dict]]]:
+    """The items under the file each varies, in order of first appearance;
+    the runs that vary none last."""
+    groups: dict[str, list[dict]] = {}
+    for item in items:
+        groups.setdefault(Path(str(item.get("variable") or "")).name, []).append(item)
+    return sorted(groups.items(), key=lambda kv: kv[0] == "")
+
+
+def folded(item: dict) -> bool:
+    """A row the block folds away: a test from before the boot oracle, or a
+    run that recorded reference frames for the oracle, which a reviewer
+    reads and a visitor does not."""
+    return item["status"] == "no verdict" or str(item.get("subject") or "").startswith("references-")
+
+
+def summary_line(items: list[dict], page_host: str) -> str:
+    """N tests, the confirmed and open counts when there are any, the last
+    run and its computer."""
+    counts = tested_counts(items)
+    parts = [f"{counts['tests']} test{'s' if counts['tests'] != 1 else ''}"]
+    parts += [f"{counts[key]} {key}" for key in ("confirmed", "open") if counts[key]]
+    parts.append(f"last run {counts['last']} on {page_host}")
+    return " · ".join(parts) + "."
 
 
 def tested_section(name: str, items: list[dict]) -> list[str]:
     """The Tested block of an emulator page, empty when nothing was recorded.
-    What every test shares is said once above the table."""
+
+    The answer first: one summary line, then the tests under the file each
+    one varies, a row per test with its result chip, a dot per variant and
+    its date. What proves it (method, claim, computer, core build, record)
+    waits in each row's details. Tests recorded before the boot oracle fold
+    under one line, and benchmarks get their own table at the end.
+    """
     if not items:
         return []
-    computer = _shared(items, lambda i: _computer(i["host"]))
-    build = _shared(items, lambda i: _build(i["core"]))
-    content = _shared(items, lambda i: i["content"] or "")
-    day = _shared(items, lambda i: i["date"][:10])
-    context = []
-    if content:
-        context.append(f"Played: {_text(content)}.")
-    if build:
-        context.append(f"Core build: {build}.")
-    if computer:
-        context.append(f"Ran on {_text(computer)}" + (f", {day}." if day else "."))
-    elif day:
-        context.append(f"Ran on {day}.")
-    rows = [tested_row(item, build is not None)
-            for item in sorted(items, key=lambda i: (i["date"], i["subject"]), reverse=True)]
-    if content:
-        rows = [siteentries.Row(r.title, r.body, ["", r.cells[1]]) for r in rows]
-    return [
-        "## Tested",
-        "",
-        f"This emulator was run in real conditions by the [core probe](../{METHOD_PAGE}). "
-        f"Each test below is the latest of its kind. "
-        f"[All measurements](../{EXPLORER_PAGE}), "
-        f"[these records as SQLite](../downloads/measurements/{name}.sqlite).",
-        "",
-        *([" ".join(context), ""] if context else []),
-        *siteentries.entry_table(rows, ["Test", "Game", "Result"], f"rb-tested-{name}", "tests"),
-    ]
+    hosts = [i.get("host_short") or i["host"] for i in items]
+    page_host = max(sorted(set(hosts)), key=hosts.count)
+    cores = [core_label(i) for i in items]
+    main_core = max(sorted(set(cores)), key=cores.count)
+    titles = distinct_titles(items)
+    links = (f'<a href="{markup.site_href(METHOD_PAGE.removesuffix(".md") + "/")}">How</a> · '
+             f'<a href="{markup.site_href(EXPLORER_PAGE.removesuffix(".md") + "/")}">All tests</a> · '
+             f'<a href="{markup.site_href(f"downloads/measurements/{name}.sqlite")}">SQLite</a>')
+    ordered = sorted(items, key=lambda i: (i["date"], i["subject"], core_label(i)), reverse=True)
+    benches = [i for i in ordered if i["kind"] == "bench"]
+    # The region runs (region_check.py) read as one outcome per game, on the
+    # system page: listed here they would be a row per run.
+    regional = [i for i in ordered if i["kind"] != "bench" and REGION_SUBJECT.match(i["subject"])]
+    ordered = [i for i in ordered if i not in regional]
+    earlier = [i for i in ordered if i["kind"] != "bench" and folded(i)]
+    current = [i for i in ordered if i["kind"] != "bench" and not folded(i)]
+    lines = ["## Tested", "", '<div class="rb-tested">',
+             f'<p class="rb-tested-summary">{_cell(summary_line(items, page_host))} {links}</p>']
+    if current or earlier:
+        lines.append('<div class="rb-tested-head" aria-hidden="true"><span>Test</span>'
+                     "<span>Result</span><span>Variants</span><span>Date</span></div>")
+    for file, group in by_file(current):
+        label = f"<code>{_cell(file)}</code>" if file else "No file varied"
+        lines.append(f'<p class="rb-tested-file">{label}</p>')
+        for item in group:
+            lines += _row(item, titles[item["id"]], page_host, main_core)
+    if current:
+        lines.append('<p class="rb-tested-legend"><span class="rb-dot rb-dot-0"></span> A dot per file tried, '
+                     'the same colour for the same result. <span class="rb-dot rb-dot-0 rb-dot-empty">'
+                     "</span> No file staged.</p>")
+    if earlier:
+        old = sum(1 for i in earlier if i["status"] == "no verdict")
+        parts = [f"{old} test{'s' if old != 1 else ''} from before the boot oracle" if old else "",
+                 (f"{len(earlier) - old} reference run{'s' if len(earlier) - old != 1 else ''}"
+                  if len(earlier) - old else "")]
+        lines += ['<details class="rb-tested-earlier">',
+                  f"<summary>{_cell(' and '.join(p for p in parts if p))}</summary>"]
+        for item in earlier:
+            file = Path(str(item.get("variable") or "")).name
+            title = f"{file}, {titles[item['id']]}" if file else titles[item["id"]]
+            lines += _row(item, title, page_host, main_core)
+        lines.append("</details>")
+    if regional:
+        games = len({i["content"] for i in regional})
+        lines.append(f'<p class="rb-tested-file">Region check: {len(regional)} runs of '
+                     f"{games} game{'s' if games != 1 else ''}, read on the system page.</p>")
+    if benches:
+        # One line: MDX reads a table split over lines as paragraphs.
+        table = ['<table class="rb-tested-bench"><thead><tr><th>Benchmark</th><th>ms per frame</th>'
+                 "<th>fps</th><th>stdev</th><th>runs</th></tr></thead><tbody>"]
+        for item in benches:
+            timing = item.get("timing") or {}
+            title = _cell(titles[item["id"]])
+            if core_label(item) != main_core:
+                title += f' <small>{_cell(core_label(item))}</small>'
+            table.append(
+                f'<tr><td>{_link(item["record_url"], title)}</td>'
+                f'<td>{timing.get("ms_per_frame", 0):.3f}</td><td>{timing.get("fps", 0):.1f}</td>'
+                f'<td>{timing.get("stdev", 0):.3f}</td><td>{item["repetitions"]}</td></tr>')
+        lines.append("".join([*table, "</tbody></table>"]))
+    return [*lines, "</div>", ""]
 
 
 def hosts_table(index: dict) -> list[str]:
     hosts: dict[str, int] = {}
     for item in index.get("items", []):
-        computer = _computer(item["host"])
+        # The load average differs per run: the computer is what comes before it.
+        computer = item["host"].split(", load ")[0]
         hosts[computer] = hosts.get(computer, 0) + 1
     if not hosts:
         return ["No result is recorded yet.", ""]
@@ -600,7 +1151,7 @@ def with_hosts(page: str, index: dict) -> str:
     return page.replace(HOSTS_MARKER, "\n".join(hosts_table(index)).rstrip("\n"))
 
 
-def generate_explorer_page(index: dict) -> str:
+def generate_explorer_page(index: dict, system_names: dict[str, str] | None = None) -> str:
     commit = index.get("commit")
     repo = index.get("repo") or ""
     source = (f"[`measurements/`]({repo}/tree/{commit}/{mr.RECORDS_DIR}) at {_short(commit)}"
@@ -623,6 +1174,7 @@ def generate_explorer_page(index: dict) -> str:
         "<MeasurementsExplorer />",
         "",
         *campaign_section(index.get("campaign")),
+        *coverage_section(index.get("coverage"), system_names),
     ])
 
 

scripts/standalone_probe.py

@@ -0,0 +1,714 @@
+#!/usr/bin/env python3
+"""Run a platform's standalone emulator on a game and record what it did.
+
+The libretro counterpart is core_probe.py: this script records the same way,
+in the same measurements/<emulator>.jsonl, for the release binaries the
+platforms install beside RetroArch.
+
+  resolve  the release asset a platform's installer picks, by its own rule
+  fetch    download it into .cache/standalone/, by sha256
+  run      one execution, printed as JSON
+  check    the same execution under several variants of one BIOS file
+
+The binary is the asset the platform downloads, unpacked from its AppImage
+without running it. Each repetition starts a private Xvfb, runs the emulator
+in core_probe's allowlist sandbox under strace, and samples the screen at a
+fixed interval from the X server's framebuffer file. The BIOS files are
+staged in a directory the emulator's settings name as its BIOS folder, the
+way the platform configures it, and the files it opened there come from
+strace.
+
+The verdict of a repetition is one of boots, bios_error, crash, hang or
+unknown, by the oracle's rules: a reference screen reached, an error string
+of the emulator in its log, the process dying, a uniform screen to the end.
+"""
+
+from __future__ import annotations
+
+import argparse
+import contextlib
+import datetime as dt
+import hashlib
+import json
+import os
+import re
+import shutil
+import signal
+import struct
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import urllib.request
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import IO
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import core_probe as cp
+
+ROOT = Path(__file__).resolve().parent.parent
+CACHE = ROOT / ".cache" / "standalone"
+DEFAULT_WORK_DIR = ROOT / "tmp" / "standalone_probe"
+RECORD_VERSION = 4
+SCREEN = (1280, 720)
+OBSERVABLES = ("verdict", "screens", "frame_hashes", "uniform_frames")
+SCRIPT = "scripts/standalone_probe.py"
+
+
+# --------------------------------------------------------------------------
+# What each emulator is, how the platform installs it, how it is driven
+# --------------------------------------------------------------------------
+
+@dataclass(frozen=True)
+class Emulator:
+    """A standalone emulator as one platform installs it.
+
+    `asset_*` reproduce the installer's choice: EmuDeck's getReleaseURLGH
+    takes the first asset, over GitHub's release list in its own order
+    (pre-releases included), whose name contains `asset_contains` and ends
+    with `asset_endswith`.
+    """
+
+    name: str
+    platform: str
+    installer_ref: str
+    repo: str
+    asset_endswith: str
+    asset_contains: str
+    binary: str
+    settings_file: str
+    error_patterns: tuple[str, ...]
+    settings: dict[str, dict[str, str]] = field(default_factory=dict)
+    bios_setting: tuple[str, str] = ("", "")
+    args: tuple[str, ...] = ()
+
+
+EMULATORS: dict[str, Emulator] = {
+    "duckstation": Emulator(
+        name="duckstation",
+        platform="emudeck",
+        installer_ref="dragoonDorise/EmuDeck functions/EmuScripts/emuDeckDuckStation.sh "
+                      "(getReleaseURLGH \"stenzek/duckstation\" \"AppImage\" \"x64.\")",
+        repo="stenzek/duckstation",
+        asset_endswith="AppImage",
+        asset_contains="x64.",
+        binary="usr/bin/duckstation-qt",
+        # Core::SetDataRoot (src/core/core.cpp): $XDG_CONFIG_HOME/duckstation on Linux.
+        settings_file="data/duckstation/settings.ini",
+        bios_setting=("BIOS", "SearchDirectory"),
+        settings={
+            "Main": {"SetupWizardIncomplete": "false", "ConfirmPowerOff": "false",
+                     "StartFullscreen": "true"},
+            "AutoUpdater": {"CheckAtStartup": "false"},
+            "GPU": {"Renderer": "Software"},
+            # No sound device in the sandbox: the null backend, not an error dialog.
+            "Audio": {"Backend": "Null"},
+            "Logging": {"LogLevel": "Info", "LogToConsole": "true", "LogToFile": "false"},
+            "Console": {"Region": "Auto"},
+        },
+        args=("-batch", "-nogui", "-fullscreen", "--"),
+        # Strings of src/core/bios.cpp and src/core/system.cpp, as the
+        # release binary carries them.
+        error_patterns=("No BIOS image found for", "No BIOS images found",
+                        "No BIOS images were found", "Failed to open BIOS"),
+    ),
+    "pcsx2": Emulator(
+        name="pcsx2",
+        platform="emudeck",
+        installer_ref="dragoonDorise/EmuDeck functions/EmuScripts/emuDeckPCSX2QT.sh "
+                      "(getReleaseURLGH \"PCSX2/pcsx2\" \"Qt.AppImage\" \"\")",
+        repo="PCSX2/pcsx2",
+        asset_endswith="Qt.AppImage",
+        asset_contains="",
+        binary="usr/bin/pcsx2-qt",
+        # EmuFolders::SetDataDirectory (pcsx2/Pcsx2Config.cpp): $XDG_CONFIG_HOME/PCSX2 on Linux,
+        # its settings in inis/.
+        settings_file="data/PCSX2/inis/PCSX2.ini",
+        bios_setting=("Folders", "Bios"),
+        settings={
+            # SETTINGS_VERSION of pcsx2/VMManager.cpp: another value resets every setting.
+            "UI": {"SettingsVersion": "1", "SetupWizardIncomplete": "false", "ConfirmShutdown": "false",
+                   "StartFullscreen": "true"},
+            "AutoUpdater": {"CheckAtStartup": "false"},
+            # GSRendererType::SW
+            "EmuCore/GS": {"Renderer": "13"},
+            # No sound device in the sandbox: the null backend, not an error dialog
+            # over the screen.
+            "SPU2/Output": {"Backend": "Null"},
+            "Logging": {"EnableSystemConsole": "true", "EnableVerbose": "false"},
+        },
+        args=("-batch", "-nogui", "-fullscreen", "-earlyconsolelog", "--"),
+        error_patterns=("requires a PlayStation 2 BIOS in order to run",
+                        "Unable to auto locate a BIOS image", "BIOS image has not been selected",
+                        "could not be read (permission denied?)"),
+    ),
+}
+
+
+# --------------------------------------------------------------------------
+# Pure helpers
+# --------------------------------------------------------------------------
+
+def select_asset(releases: Sequence[dict], endswith: str, contains: str) -> dict | None:
+    """The asset EmuDeck's getReleaseURLGH returns: the first, over the
+    releases in the API's order, whose name contains and ends as asked."""
+    for release in releases:
+        for asset in release.get("assets") or ():
+            name = asset.get("name", "")
+            if contains in name and name.endswith(endswith):
+                return {"tag": release.get("tag_name"), "name": name,
+                        "url": asset.get("browser_download_url"),
+                        "prerelease": bool(release.get("prerelease"))}
+    return None
+
+
+def appimage_offset(head: bytes) -> int:
+    """Where an AppImage's squashfs starts: the end of its ELF runtime,
+    read from the section header table, as the runtime computes it."""
+    if head[:4] != b"\x7fELF":
+        raise cp.ProbeError("not an ELF file, not an AppImage")
+    if head[4] == 2:
+        shoff = struct.unpack_from("<Q", head, 0x28)[0]
+        shentsize, shnum = struct.unpack_from("<HH", head, 0x3A)
+    else:
+        shoff = struct.unpack_from("<I", head, 0x20)[0]
+        shentsize, shnum = struct.unpack_from("<HH", head, 0x2E)
+    return shoff + shentsize * shnum
+
+
+def ini_text(sections: dict[str, dict[str, str]]) -> str:
+    out = []
+    for section, values in sections.items():
+        out.append(f"[{section}]")
+        out.extend(f"{key} = {value}" for key, value in values.items())
+        out.append("")
+    return "\n".join(out)
+
+
+def emulator_settings(emu: Emulator, bios_dir: Path, overrides: Sequence[tuple[str, str]]
+                      ) -> dict[str, dict[str, str]]:
+    """The settings written before a run: the emulator's own, the BIOS
+    folder, then each SECTION.KEY=VALUE override."""
+    sections = {name: dict(values) for name, values in emu.settings.items()}
+    section, key = emu.bios_setting
+    sections.setdefault(section, {})[key] = str(bios_dir)
+    for name, value in overrides:
+        sec, _, k = name.rpartition(".")
+        if not sec or not k:
+            raise cp.ProbeError(f"--setting {name}: SECTION.KEY expected")
+        sections.setdefault(sec, {})[k] = value
+    return sections
+
+
+def parse_xwd(data: bytes) -> tuple[int, int, bytes]:
+    """Width, height and RGB pixels of an XWD image, the file Xvfb keeps
+    its screen in: 32 bits per pixel, least significant byte first."""
+    if len(data) < 100:
+        raise cp.ProbeError("framebuffer file shorter than an XWD header")
+    fields = struct.unpack_from(">25I", data, 0)
+    header_size, version = fields[0], fields[1]
+    width, height = fields[4], fields[5]
+    byte_order, bits_per_pixel, bytes_per_line, ncolors = fields[7], fields[11], fields[12], fields[19]
+    if version != 7 or bits_per_pixel != 32 or byte_order != 0:
+        raise cp.ProbeError(f"XWD version {version}, {bits_per_pixel} bpp, byte order "
+                            f"{byte_order}: only version 7, 32 bpp, LSB first are read")
+    start = header_size + ncolors * 12
+    rows = [data[start + y * bytes_per_line:start + y * bytes_per_line + width * 4] for y in range(height)]
+    packed = b"".join(rows)
+    if len(packed) != width * height * 4:
+        raise cp.ProbeError("framebuffer file shorter than its header says")
+    return width, height, cp.to_rgb(1, width, height, packed)
+
+
+
+# --------------------------------------------------------------------------
+# Release binaries
+# --------------------------------------------------------------------------
+
+def github_releases(repo: str) -> list[dict]:
+    from scraper.base_scraper import github_headers  # noqa: PLC0415
+
+    request = urllib.request.Request(f"https://api.github.com/repos/{repo}/releases",
+                                     headers=github_headers())
+    with urllib.request.urlopen(request, timeout=60) as response:
+        return json.load(response)
+
+
+def fetch(emu: Emulator, cache: Path = CACHE) -> dict:
+    """Download the asset the platform picks, unpack it, and describe it."""
+    asset = select_asset(github_releases(emu.repo), emu.asset_endswith, emu.asset_contains)
+    if not asset:
+        raise cp.ProbeError(f"{emu.repo}: no release asset matches the {emu.platform} rule")
+    folder = cache / emu.name
+    folder.mkdir(parents=True, exist_ok=True)
+    partial = folder / f"{asset['name']}.part"
+    with urllib.request.urlopen(asset["url"], timeout=600) as response, partial.open("wb") as out:
+        shutil.copyfileobj(response, out)
+    sha256 = cp.file_digest(partial, "sha256")
+    target = folder / sha256 / asset["name"]
+    target.parent.mkdir(parents=True, exist_ok=True)
+    partial.replace(target)
+    unpack(target)
+    meta = {**asset, "sha256": sha256, "repo": emu.repo, "platform": emu.platform,
+            "installer_ref": emu.installer_ref, "fetched_at": now_iso()}
+    (target.parent / "asset.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+    return meta
+
+
+def unpack(appimage: Path) -> Path:
+    """The AppImage's tree, unpacked with unsquashfs: the runtime is never run."""
+    tree = appimage.parent / "tree"
+    if tree.is_dir():
+        return tree
+    unsquashfs = shutil.which("unsquashfs")
+    if not unsquashfs:
+        raise cp.ProbeError("unsquashfs not found (squashfs-tools)")
+    with appimage.open("rb") as handle:
+        offset = appimage_offset(handle.read(64))
+    scratch = appimage.parent / "tree.part"
+    shutil.rmtree(scratch, ignore_errors=True)
+    subprocess.run([unsquashfs, "-q", "-n", "-o", str(offset), "-d", str(scratch), str(appimage)],
+                   check=True, stdout=subprocess.DEVNULL)
+    scratch.replace(tree)
+    return tree
+
+
+def installed(emu: Emulator, sha256: str | None, cache: Path = CACHE) -> tuple[dict, Path]:
+    """The fetched asset to run: the one named by sha256, else the newest."""
+    folder = cache / emu.name
+    metas = sorted(folder.glob("*/asset.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+    for path in metas:
+        meta = json.loads(path.read_text(encoding="utf-8"))
+        if sha256 in (None, meta["sha256"]):
+            return meta, unpack(path.parent / meta["name"])
+    raise cp.ProbeError(f"{emu.name}: no fetched asset{' ' + sha256 if sha256 else ''}, "
+                        f"run `standalone_probe.py fetch {emu.name}`")
+
+
+# --------------------------------------------------------------------------
+# One repetition
+# --------------------------------------------------------------------------
+
+def now_iso() -> str:
+    return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
+
+
+# The X server runs inside the sandbox, on the sandbox's own /tmp: the
+# display is private to the run, and the host's /tmp/.X11-unix (read-only
+# under WSLg) is never touched. Its fbdir option keeps the screen as an XWD
+# file in the run's scratch folder, which the probe reads from outside.
+DISPLAY_NUMBER = 1
+XVFB_WRAPPER = (
+    f'Xvfb ":{DISPLAY_NUMBER}" -screen 0 {SCREEN[0]}x{SCREEN[1]}x24 -nolisten tcp -fbdir "$1" '
+    '-noreset >/dev/null 2>&1 &\n'
+    'i=0\n'
+    f'while [ ! -S /tmp/.X11-unix/X{DISPLAY_NUMBER} ]; do\n'
+    '  i=$((i + 1)); [ "$i" -gt 400 ] && exit 97; sleep 0.05\n'
+    'done\n'
+    'shift\n'
+    'exec "$@"\n'
+)
+
+
+def with_display(argv: list[str], fbdir: Path) -> list[str]:
+    """The emulator's command, started once the sandbox's Xvfb listens."""
+    return ["/bin/sh", "-c", XVFB_WRAPPER, "xvfb", str(fbdir), *argv]
+
+
+def stop(proc: subprocess.Popen) -> None:
+    if proc.poll() is None:
+        with contextlib.suppress(ProcessLookupError):
+            os.killpg(proc.pid, signal.SIGKILL)
+    proc.wait()
+
+
+ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
+
+
+def read_lines(stream: IO[bytes], out: list[dict], samples: list) -> None:
+    """The emulator's console, line by line, each with the sample count
+    reached when it was written, colour escapes removed."""
+    for raw in iter(stream.readline, b""):
+        text = ANSI.sub("", raw.decode("utf-8", errors="replace")).rstrip()
+        if text:
+            out.append({"frame": len(samples), "level": "info", "text": text})
+
+
+def run_once(args: argparse.Namespace, emu: Emulator, tree: Path, system_dir: Path, scratch: Path,
+             png_dir: Path | None) -> dict:
+    home = scratch / "home"
+    data = scratch / "data"
+    for folder in (home, data):
+        folder.mkdir(parents=True, exist_ok=True)
+    settings = emulator_settings(emu, system_dir.resolve(), args.setting)
+    settings_path = scratch / emu.settings_file
+    settings_path.parent.mkdir(parents=True, exist_ok=True)
+    settings_path.write_text(ini_text(settings), encoding="utf-8")
+    content = Path(args.content).resolve()
+    argv = [str(tree / emu.binary)] + [a.format(data=data.resolve()) for a in emu.args] + [str(content)]
+    trace = scratch / "strace.txt"
+    strace = shutil.which("strace") if not args.no_trace else None
+    if strace:
+        argv = [strace, "-f", "-qq", "-s", "4096", "-e", "trace=openat,open", "-o", str(trace)] + argv
+    samples: list[tuple[float, str, bool]] = []
+    logs: list[dict] = []
+    if not shutil.which("Xvfb"):
+        raise cp.ProbeError("Xvfb not found")
+    fbdir = scratch / "fb"
+    fbdir.mkdir(parents=True, exist_ok=True)
+    screen_file = fbdir / "Xvfb_screen0"
+    env = {"HOME": str(home.resolve()), "XDG_CONFIG_HOME": str(data.resolve()),
+           "XDG_DATA_HOME": str(data.resolve()), "XDG_CACHE_HOME": str((home / ".cache").resolve()),
+           "DISPLAY": f":{DISPLAY_NUMBER}", "QT_QPA_PLATFORM": "xcb", "LIBGL_ALWAYS_SOFTWARE": "1",
+           "PATH": "/usr/bin:/bin", "LANG": "C.UTF-8"}
+    # The BIOS folder is writable, as the platform's is: PCSX2 creates its
+    # NVRAM and MEC files there.
+    prefix, isolation = cp.sandbox_argv([scratch, system_dir], [tree, content.parent])
+    if args.require_sandbox and isolation != cp.SANDBOX:
+        raise cp.ProbeError(f"refused to run the emulator outside a sandbox: {isolation}")
+    proc = subprocess.Popen(prefix + with_display(argv, fbdir.resolve()), stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT, env=env, start_new_session=True)
+    reader = threading.Thread(target=read_lines, args=(proc.stdout, logs, samples), daemon=True)
+    reader.start()
+    try:
+        deadline = time.monotonic() + 30
+        while not screen_file.exists():
+            if proc.poll() is not None or time.monotonic() > deadline:
+                raise cp.ProbeError(f"Xvfb did not start in the sandbox (exit {proc.poll()}): "
+                                    f"{' / '.join(line['text'] for line in logs[-5:])}")
+            time.sleep(0.05)
+        started = time.monotonic()
+        ended = None
+        for index in range(args.samples):
+            target = started + (index + 1) * args.interval
+            while time.monotonic() < target and proc.poll() is None:
+                time.sleep(min(0.05, max(0.0, target - time.monotonic())))
+            if proc.poll() is not None:
+                code = proc.returncode
+                ended = {"status": code if code >= 0 else None,
+                         "signal": signal.Signals(-code).name if code < 0 else None}
+                break
+            width, height, rgb = parse_xwd(screen_file.read_bytes())
+            sha1 = hashlib.sha1(rgb).hexdigest()
+            samples.append((round(time.monotonic() - started, 3), sha1, cp.uniform_frame(rgb, 3)))
+            if png_dir and not any(p.name.endswith(f"{sha1[:12]}.png") for p in png_dir.iterdir()):
+                (png_dir / f"{index:04d}-{sha1[:12]}.png").write_bytes(cp.png_bytes(width, height, rgb))
+        wall = time.monotonic() - started
+    finally:
+        stop(proc)
+        reader.join(5)
+    places = [(str(system_dir.resolve()), "<bios>"), (str(scratch.resolve()), "<scratch>"),
+              (str(content.parent), "<content>"), (str(tree), "<emulator>"), (str(ROOT), "<repo>")]
+    for line in logs:
+        line["text"] = cp.host_paths_hidden(line["text"], places)
+    rep: dict = {
+        "loaded": ended is None,
+        "library_name": emu.name,
+        "frames": max(len(samples), 1),
+        "wall_s": round(wall, 3),
+        "sample_interval_s": args.interval,
+        "sample_times_s": [t for t, _, _ in samples],
+        # The shapes core_probe gives them: [sample, sha1] pairs, and the
+        # digests of the samples that are one colour.
+        "observables": {
+            "frame_hashes": [[index, h] for index, (_, h, _) in enumerate(samples)],
+            "uniform_frames": sorted({h for _, h, u in samples if u}),
+            "screens": sorted({h for _, h, u in samples if not u}),
+        },
+        "logs": logs,
+        "isolation": isolation,
+    }
+    if ended:
+        rep["exit"] = ended
+    if strace and trace.is_file():
+        opened, probed = cp.files_under(cp.parse_strace(trace.read_text(encoding="utf-8", errors="replace")),
+                                        system_dir)
+        rep["files_opened"] = [{"path": rel, "sha1_after": cp.file_digest(system_dir / rel)}
+                               for rel in opened if (system_dir / rel).is_file()]
+        rep["files_probed_absent"] = probed
+        rep["tracing"] = "strace"
+    else:
+        rep["files_opened"] = None
+        rep["files_probed_absent"] = None
+        rep["tracing"] = "disabled: --no-trace" if args.no_trace else "unavailable: strace not found"
+    return rep
+
+
+# --------------------------------------------------------------------------
+# Variants and the record
+# --------------------------------------------------------------------------
+
+def run_variant(args: argparse.Namespace, emu: Emulator, tree: Path, oracle: dict,
+                base: dict[str, Path], variant: cp.Variant, work: Path, repetitions: int) -> dict:
+    reps: list[dict] = []
+    flags: list[str] = []
+    for index in range(repetitions):
+        # A fresh BIOS folder each time: PCSX2 writes its NVRAM and MEC files
+        # beside the BIOS, as on a user's folder, and a repetition must not
+        # start from what the last one wrote.
+        system_dir = work / f"bios{index}"
+        staged = cp.stage_system_dir(base, variant, system_dir)
+        png_dir = None
+        if args.png_dir and index == 0:
+            png_dir = Path(args.png_dir) / variant.label
+            png_dir.mkdir(parents=True, exist_ok=True)
+        rep = run_once(args, emu, tree, system_dir, work / f"rep{index}", png_dir)
+        rep["verdict"], rep["verdict_evidence"] = cp.judge(rep, oracle)
+        rep["observables"]["verdict"] = rep["verdict"]
+        for entry in rep.get("files_opened") or []:
+            expected = staged.get(entry["path"])
+            entry["sha1_staged"] = expected
+            if expected is not None and expected != entry["sha1_after"]:
+                flags.append(f"repetition {index}: {entry['path']} sha1 changed during the run")
+        reps.append(rep)
+    verdict, evidence = cp.variant_verdict(reps)
+    return {"label": variant.label, "system_files": dict(sorted(staged.items())), "repetitions": reps,
+            "flags": flags, "verdict": verdict, "verdict_evidence": evidence}
+
+
+def frontend_identity() -> dict:
+    script = Path(__file__).resolve()
+    commit = dirty = None
+    try:
+        commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, text=True,
+                                check=True).stdout.strip()
+        dirty = bool(subprocess.run(["git", "status", "--porcelain", "--", str(script)], cwd=ROOT,
+                                    capture_output=True, text=True, check=True).stdout.strip())
+    except (OSError, subprocess.CalledProcessError) as exc:
+        print(f"WARNING: frontend commit unknown, git failed: {exc}", file=sys.stderr)
+    return {"script": SCRIPT, "commit": commit, "script_modified": dirty,
+            "script_sha256": cp.file_digest(script, "sha256")}
+
+
+def emulator_oracle(emu: Emulator, path: Path, content_sha1: str) -> dict:
+    """The oracle core_probe reads (tests/campaign/oracle.yml), looked up by
+    the content's sha1 and the emulator's name, with the emulator's own
+    error strings of this script added as patterns. Without the file the
+    oracle names this script, which then holds every pattern."""
+    data, identity = cp.load_oracle(path)
+    entries = cp.oracle_for(data, content_sha1, emu.name)
+    own = [re.escape(text) for text in emu.error_patterns]
+    entries["error_patterns"] = list(dict.fromkeys(entries["error_patterns"] + own))
+    if identity is None:
+        identity = {"file": SCRIPT, "sha256": cp.file_digest(Path(__file__).resolve(), "sha256")}
+    return {**identity, **entries}
+
+
+def check_result(variants: list[dict], observable: str) -> dict:
+    """core_probe's comparison, with each variant's verdict beside it."""
+    return {**cp.check_result(variants, observable),
+            "verdicts": {v["label"]: v["verdict"] for v in variants}}
+
+
+def probe(args: argparse.Namespace) -> int:
+    emu = EMULATORS[args.emulator]
+    problems = validate(args)
+    if problems:
+        for problem in problems:
+            print(f"ERROR: {problem}", file=sys.stderr)
+        return 2
+    # One run on the host at a time, as core_probe: the lock is held for
+    # this run, its load check included, and freed when it ends.
+    with cp.host_probe_lock():
+        return _probe(args, emu)
+
+
+def _probe(args: argparse.Namespace, emu: Emulator) -> int:
+    meta, tree = installed(emu, args.sha256)
+    max_load = args.max_load if args.max_load is not None else (os.cpu_count() or 1) / 4
+    load_before = cp.load_average()
+    if args.record and load_before and load_before[0] > max_load:
+        print(f"ERROR: load average {load_before[0]} exceeds {max_load}: the machine is busy, "
+              f"nothing recorded", file=sys.stderr)
+        return 3
+    oracle = emulator_oracle(emu, Path(args.oracle) if args.oracle else cp.DEFAULT_ORACLE,
+                             cp.file_digest(Path(args.content)))
+    repetitions = 1 if args.mode == "run" else args.repeat
+    base = {dest: Path(src) for dest, src in args.bios_file}
+    if args.mode == "check":
+        variants_in = []
+        seen: set[str] = set()
+        for dest, src in args.vary:
+            label = Path(src).name if src else f"{dest}-absent"
+            while label in seen:
+                label += "+"
+            seen.add(label)
+            variants_in.append(cp.Variant(label, {dest: Path(src) if src else None}))
+    else:
+        variants_in = [cp.Variant("default")]
+    work = Path(tempfile.mkdtemp(prefix=f"{args.mode}-", dir=cp._ensure(Path(args.work_dir))))
+    try:
+        variants = [run_variant(args, emu, tree, oracle, base, variant, work / f"v{i}", repetitions)
+                    for i, variant in enumerate(variants_in)]
+    except (cp.ProbeError, OSError, subprocess.SubprocessError) as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 1
+    finally:
+        shutil.rmtree(work, ignore_errors=True)
+    load_after = cp.load_average()
+    record: dict = {
+        "record_version": RECORD_VERSION,
+        "emulator": emu.name,
+        "subject": None,
+        "supersedes": args.supersedes,
+        "kind": args.mode,
+        "recorded_at": now_iso(),
+        "core": {"file": meta["name"], "sha256": meta["sha256"], "repo": f"https://github.com/{meta['repo']}",
+                 "commit": None, "build_command": None, "compiler": None, "library_name": emu.name,
+                 "library_version": meta["tag"], "artifact": meta["url"]},
+        "frontend": frontend_identity(),
+        "host": {**cp.host_identity(), "load_before": load_before, "load_after": load_after,
+                 "max_load": max_load},
+        "inputs": {
+            "content": cp.content_identity(Path(args.content), args.content_label),
+            "options": {f"{s}.{k}": v for s, values in
+                        emulator_settings(emu, Path("<bios>"), args.setting).items() for k, v in values.items()},
+            "env": {},
+            "frames": args.samples,
+            "warmup": 0,
+            "repetitions": repetitions,
+            "presses": [],
+            "observables": list(OBSERVABLES),
+            "frame_hash_every": 1,
+            "joypad_ports": 0,
+            "oracle": oracle,
+        },
+        "variants": variants,
+        "flags": [f"{v['label']}: {flag}" for v in variants for flag in v["flags"]],
+    }
+    if load_before and load_after and max(load_before[0], load_after[0]) > max_load:
+        record["flags"].append(f"load average reached {max(load_before[0], load_after[0])} "
+                               f"(threshold {max_load})")
+    if args.mode == "check":
+        record["check"] = {"claim": args.claim, "method": args.method, "observable": args.observable,
+                           "variable": args.vary[0][0], "result": check_result(variants, args.observable)}
+    from measurement_records import canonical, sha256_hex, slugify
+
+    if args.subject:
+        record["subject"] = args.subject
+    elif args.mode == "check":
+        digest = sha256_hex(canonical([args.claim, args.method, args.observable, args.vary[0][0]]))[:8]
+        record["subject"] = f"{slugify(args.claim)}-{digest}"
+    else:
+        digest = sha256_hex(canonical([meta["sha256"], record["inputs"]["content"],
+                                       record["inputs"]["options"], args.bios_file]))[:8]
+        record["subject"] = f"run-{slugify(args.content_label or Path(args.content).name, 30)}-{digest}"
+    if args.record:
+        try:
+            lines, record = cp.write_record(Path(args.record), record)
+        except cp.ProbeError as exc:
+            print(f"ERROR: {exc}", file=sys.stderr)
+            return 2
+        print(f"recorded {lines}", file=sys.stderr)
+    print(json.dumps(record, indent=2))
+    return 0
+
+
+def validate(args: argparse.Namespace) -> list[str]:
+    problems = []
+    if not Path(args.content).is_file():
+        problems.append(f"--content {args.content}: not a file")
+    for dest, src in args.bios_file:
+        if not Path(src).is_file():
+            problems.append(f"--bios-file {dest}={src}: not a file")
+    if args.oracle and not Path(args.oracle).is_file():
+        problems.append(f"--oracle {args.oracle}: not a file")
+    if args.samples < 2 or args.interval <= 0:
+        problems.append("--samples must be >= 2 and --interval > 0")
+    if args.require_sandbox:
+        isolation = cp.sandbox_argv([])[1]
+        if isolation != cp.SANDBOX:
+            problems.append(f"--require-sandbox: {isolation}")
+    if args.record:
+        wrong = cp.not_a_checkout_root(Path(args.record))
+        if wrong:
+            problems.append(wrong)
+    if args.subject and not re.fullmatch(r"[a-z0-9]+(?:[-_][a-z0-9]+)*", args.subject):
+        problems.append(f"--subject {args.subject!r}: lowercase letters, digits, - and _ only")
+    if args.mode == "check":
+        if len({dest for dest, _ in args.vary}) != 1:
+            problems.append("--vary must name the same DEST in every occurrence")
+        for dest, src in args.vary:
+            if src and not Path(src).is_file():
+                problems.append(f"--vary {dest}={src}: not a file")
+        if args.observable not in OBSERVABLES:
+            problems.append(f"--observable {args.observable}: not one of {', '.join(OBSERVABLES)}")
+    return problems
+
+
+# --------------------------------------------------------------------------
+# Command line
+# --------------------------------------------------------------------------
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    sub = parser.add_subparsers(dest="mode", required=True)
+    for name, text in (("resolve", "print the asset the platform's installer picks"),
+                       ("fetch", "download and unpack it into .cache/standalone/")):
+        p = sub.add_parser(name, help=text)
+        p.add_argument("emulator", choices=sorted(EMULATORS), help="the emulator, by its profile name")
+    common = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
+    common.add_argument("emulator", choices=sorted(EMULATORS), help="the emulator, by its profile name")
+    common.add_argument("--content", required=True, help="the game: a .cue, .iso or .chd")
+    common.add_argument("--content-label", help="identity of the content, its Redump or No-Intro name")
+    common.add_argument("--bios-file", action="append", default=[], metavar="DEST=SRC",
+                        type=lambda t: cp.parse_assignment(t, "--bios-file"),
+                        help="copy SRC into the staged BIOS folder as DEST")
+    common.add_argument("--setting", action="append", default=[], metavar="SECTION.KEY=VALUE",
+                        type=lambda t: cp.parse_assignment(t, "--setting"),
+                        help="an emulator setting written before the run")
+    common.add_argument("--sha256", help="the fetched asset to run (default: the newest fetched)")
+    common.add_argument("--samples", type=int, default=60, help="screen samples per repetition")
+    common.add_argument("--interval", type=float, default=1.0, help="seconds between two samples")
+    common.add_argument("--oracle", metavar="FILE",
+                        help="the boot oracle (default: tests/campaign/oracle.yml, as core_probe)")
+    common.add_argument("--png-dir", help="write each distinct sampled screen of repetition 0 as PNG")
+    common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
+                        help="append the record to measurements/<emulator>.jsonl in the checkout DIR")
+    common.add_argument("--subject", help="slug naming what the record tests")
+    common.add_argument("--supersedes", metavar="ID", help="id of an earlier record this one corrects")
+    common.add_argument("--max-load", type=float, default=None,
+                        help="refuse to record above this 1-minute load average (default: CPUs / 4)")
+    common.add_argument("--work-dir", default=str(DEFAULT_WORK_DIR), help="scratch directory")
+    common.add_argument("--no-trace", action="store_true", help="do not run under strace")
+    common.add_argument("--require-sandbox", action="store_true",
+                        help="refuse to run unless bubblewrap can isolate the emulator")
+    sub.add_parser("run", parents=[common], help="one execution")
+    check = sub.add_parser("check", parents=[common], help="compare variants of one BIOS file")
+    check.add_argument("--vary", action="append", required=True, metavar="DEST=SRC",
+                       type=lambda t: cp.parse_assignment(t, "--vary"),
+                       help="one variant per occurrence; an empty SRC leaves DEST absent")
+    check.add_argument("--observable", default="verdict", help="compared across variants")
+    check.add_argument("--claim", required=True, help="the statement under test")
+    check.add_argument("--method", required=True, help="how the run tests it")
+    check.add_argument("--repeat", type=int, default=2, help="runs of each variant")
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    emu = EMULATORS[args.emulator]
+    if args.mode == "resolve":
+        asset = select_asset(github_releases(emu.repo), emu.asset_endswith, emu.asset_contains)
+        print(json.dumps({"emulator": emu.name, "platform": emu.platform,
+                          "installer_ref": emu.installer_ref, "asset": asset}, indent=2))
+        return 0 if asset else 1
+    if args.mode == "fetch":
+        try:
+            print(json.dumps(fetch(emu), indent=2))
+        except (cp.ProbeError, OSError, subprocess.CalledProcessError) as exc:
+            print(f"ERROR: {exc}", file=sys.stderr)
+            return 1
+        return 0
+    return probe(args)
+
+
+if __name__ == "__main__":
+    from common import run_cli
+
+    run_cli(main)