feat: settle profile fields from boot verdicts

The campaign runs the full set, no BIOS, and per file the file alone with zeros, its first half and other regions' dumps under its name. required, hle_fallback and validation are read from the verdicts, never from frames that differ. Benchmarks run only with --bench. The timer command is gone: a maintainer starts each pass. references lists stable screens for the oracle.

scripts/campaign_rules.py

@@ -1,25 +1,36 @@
-"""The campaign's fixed rule table: which observable settles which profile field.
+"""The campaign's fixed rule table: which verdicts settle which profile field.
 
-Each rule is a pure function of a core probe record (or what was read from
-one) and a profile entry. It returns verdicts. A verdict with `apply` set is
-settled by the table and written into the profile with the record ids as
-`evidence:`. Any other verdict goes to the review queue
-(measurements/findings.jsonl).
+Each rule is a pure function of verdicts read from core probe records (the
+boot oracle of core_probe.py: boots, bios_error, crash, hang, unknown) and
+a profile entry. It returns verdicts on profile fields. One with `apply` set
+is settled by the table and written into the profile with the record ids as
+`evidence:`. Any other goes to the review queue (measurements/findings.jsonl).
 
-  observable                                        field          settled
-  boots to the game with no file                    hle_fallback   true
-  no game screen without it, the game with it       required       true
-                                                    hle_fallback   false
-  opens the staged file under another path          path           that path
-  opens the staged file under another name          aliases        name added
-  a staged file the content should use, never read  -              flagged only
+Three runs are compared for a file, all on the same content: `none` with no
+BIOS at all, `present` with the file alone, and wrong files staged under its
+name (zeros of its size, its first half, another region's dump).
 
-"Boots to the game" is read from the sampled frames: the run without the
-file shows at least MIN_SHARED_SCREENS distinct screens that the run with
-the file also shows, single-colour frames excluded. "No game screen" is a
-run that did not load, or whose every sampled frame is a single colour. A
-run that shows screens of its own that the reference never shows settles
-nothing.
+  runs                                          field          settled
+  none boots                                    hle_fallback   true
+  none fails, present boots                     required       true
+                                                hle_fallback   false
+  zeros refused, present boots                  validation     a content check
+  zeros used, present boots                     validation     no content check
+  half refused, zeros used                      validation     size
+  half used                                     validation     no size check
+  opens the staged file under another path      path           that path
+  opens the staged file under another name      aliases        name added
+  a staged file the content should use, never   -              flagged only
+  read
+  copies of one file that boot and fail         -              queued
+
+"Fails" is bios_error, crash or hang. A wrong file is refused when the
+core reports a BIOS error, or when it boots without the BIOS screens the
+right file shows (it fell back to a built-in BIOS). It is used when it
+turns the run into a hang or a crash that the run without any file does not
+show. Anything else settles nothing. Of validation, the table writes only a
+missing `size`. A contradicted content check goes to review: which hash a
+core computes is read in its code.
 
 Standard library only.
 """
@@ -29,23 +40,34 @@ from __future__ import annotations
 from collections.abc import Iterable, Sequence
 from dataclasses import dataclass, field
 
-MIN_SHARED_SCREENS = 2
 BOOTS = "boots"
-FAILS = "fails"
-INCONCLUSIVE = "inconclusive"
+BIOS_ERROR = "bios_error"
+CRASH = "crash"
+HANG = "hang"
+UNKNOWN = "unknown"
+FAILS = (BIOS_ERROR, CRASH, HANG)
+REFUSED = "refused"
+USED = "used"
+CONTENT_CHECKS = ("md5", "sha1", "sha256", "crc32", "adler32")
 
 # A field an entry leaves out reads as this value.
-FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": []}
-SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases")
+FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": [],
+                                     "validation": []}
+SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases", "validation")
 
 
 @dataclass(frozen=True)
-class Boot:
-    """What the runs without the file showed, against the runs with it."""
+class Run:
+    """The verdict of one variant of a record: what a rule reads."""
 
-    status: str
-    detail: str
-    shared: int = 0
+    verdict: str
+    record_id: str
+    label: str
+    stages: tuple[str, ...] = ()
+
+    @property
+    def shows_bios(self) -> bool:
+        return "bios" in self.stages
 
 
 @dataclass(frozen=True)
@@ -67,13 +89,6 @@ class Opened:
     extra: dict = field(default_factory=dict)
 
 
-def screens(repetition: dict) -> set[str]:
-    """The distinct sampled frames of a repetition that are not one colour."""
-    observables = repetition.get("observables") or {}
-    uniform = set(observables.get("uniform_frames") or ())
-    return {digest for _, digest in observables.get("frame_hashes") or () if digest not in uniform}
-
-
 def variant(record: dict, label: str) -> dict:
     for item in record.get("variants") or ():
         if item.get("label") == label:
@@ -81,43 +96,16 @@ def variant(record: dict, label: str) -> dict:
     raise KeyError(f"record {record.get('id')}: no variant {label!r}")
 
 
-def boot_outcome(record: dict, absent_label: str, present_label: str) -> Boot:
-    """Whether the core reached the game without the file.
+def run_of(record: dict, label: str) -> Run:
+    """A variant's verdict as a rule reads it. A record from before the
+    oracle has none, and reads as unknown."""
+    item = variant(record, label)
+    evidence = item.get("verdict_evidence") or {}
+    return Run(item.get("verdict", UNKNOWN), record["id"], label, tuple(evidence.get("stages") or ()))
 
-    Every repetition must agree: one that boots and one that does not leave
-    the question open.
-    """
-    present = variant(record, present_label)["repetitions"]
-    if any(not rep.get("loaded") for rep in present):
-        return Boot(INCONCLUSIVE, "with the file, the core did not load the content")
-    reference: set[str] = set()
-    for rep in present:
-        shown = screens(rep)
-        if not shown:
-            return Boot(INCONCLUSIVE, "with the file, the core showed no game screen")
-        reference |= shown
-    outcomes: list[str] = []
-    shared_counts: list[int] = []
-    for rep in variant(record, absent_label)["repetitions"]:
-        if not rep.get("loaded"):
-            outcomes.append(FAILS)
-            continue
-        shown = screens(rep)
-        if not shown:
-            outcomes.append(FAILS)
-            continue
-        shared = len(shown & reference)
-        shared_counts.append(shared)
-        outcomes.append(BOOTS if shared >= MIN_SHARED_SCREENS else INCONCLUSIVE)
-    shared = min(shared_counts) if shared_counts else 0
-    if all(o == BOOTS for o in outcomes):
-        return Boot(BOOTS, f"without the file, every run showed {shared} or more screens the "
-                           f"run with the file shows", shared)
-    if all(o == FAILS for o in outcomes):
-        return Boot(FAILS, "without the file, no run loaded the content or showed more than "
-                           "one colour, while the run with the file reached the game")
-    return Boot(INCONCLUSIVE, "without the file, the runs showed screens the run with the file "
-                              "never shows, or disagreed with each other", shared)
+
+def ids(*runs: Run) -> tuple[str, ...]:
+    return tuple(dict.fromkeys(run.record_id for run in runs))
 
 
 def current(entry: dict, name: str) -> object:
@@ -130,32 +118,110 @@ def agrees(entry: dict, verdict: Verdict) -> bool:
     """True when the profile already says what the run showed."""
     if verdict.field == "aliases":
         return set(verdict.value or ()) <= set(entry.get("aliases") or ())
+    if verdict.field == "validation":
+        return verdict.rule == "validation_agrees"
     return current(entry, verdict.field) == verdict.value
 
 
-def rule_bios_need(boot: Boot, entry: dict, record_id: str) -> list[Verdict]:
-    """Boot with no file against boot with the declared file."""
+def rule_need(none: Run, present: Run, entry: dict) -> list[Verdict]:
+    """No BIOS at all against the file alone: required and hle_fallback."""
     name = entry["name"]
-    evidence = (record_id,)
-    if boot.status == BOOTS:
+    if none.verdict == BOOTS:
         out = [Verdict("boots_without_file", "hle_fallback", True,
-                       f"the core reaches the game without {name}: {boot.detail}", evidence, True)]
+                       f"the core boots the content with no BIOS at all, {name} included",
+                       ids(none), True)]
         if entry.get("required") is True:
             out.append(Verdict(
                 "boots_without_file", "required", False,
-                f"the profile says {name} is required, and the core reaches the game "
-                f"without it. The table settles hle_fallback from this run, not required",
-                evidence))
+                f"the profile says {name} is required, and the core boots without it. The table "
+                f"settles hle_fallback from this run, not required", ids(none)))
         return out
-    if boot.status == FAILS:
+    if none.verdict in FAILS and present.verdict == BOOTS:
         return [
             Verdict("fails_without_file", "required", True,
-                    f"the core reaches the game only with {name}: {boot.detail}", evidence, True),
+                    f"with no BIOS the run ends in {none.verdict}, with {name} alone it boots",
+                    ids(none, present), True),
             Verdict("fails_without_file", "hle_fallback", False,
-                    f"no stand-in for {name}: {boot.detail}", evidence, True),
+                    f"no stand-in for {name}: with no BIOS the run ends in {none.verdict}",
+                    ids(none, present), True),
         ]
-    return [Verdict("inconclusive", "hle_fallback", None,
-                    f"the runs with and without {name} settle nothing: {boot.detail}", evidence)]
+    return [Verdict("unsettled", "hle_fallback", None,
+                    f"with no BIOS the run is {none.verdict} and with {name} alone {present.verdict}: "
+                    f"no rule settles a field from that", ids(none, present))]
+
+
+def file_use(wrong: Run, none: Run, present: Run) -> str | None:
+    """Whether the core refused a wrong file or ran it, or None."""
+    if present.verdict != BOOTS:
+        return None
+    if wrong.verdict == BIOS_ERROR:
+        return REFUSED
+    if wrong.verdict == BOOTS:
+        if wrong.shows_bios:
+            return USED
+        return REFUSED if present.shows_bios else None
+    if wrong.verdict in (HANG, CRASH) and none.verdict in (BOOTS, BIOS_ERROR):
+        return USED
+    return None
+
+
+def declared_checks(entry: dict) -> tuple[set[str], bool]:
+    """The checks the entry's validation names, and whether it is one list
+    (a per-build dict is never edited by the table)."""
+    value = entry.get("validation") or []
+    if isinstance(value, dict):
+        return {str(v) for items in value.values() for v in items or ()}, False
+    return {str(v) for v in value}, True
+
+
+def rule_validation(zeros: Run | None, half: Run | None, none: Run, present: Run,
+                    entry: dict) -> Verdict | None:
+    """Same-size and short wrong files against the right one: validation."""
+    name = entry["name"]
+    content = {REFUSED: True, USED: False}.get(file_use(zeros, none, present)) if zeros else None
+    half_use = file_use(half, none, present) if half else None
+    size: bool | None = None
+    if half_use == USED:
+        size, content = False, False if content is None else content
+    elif half_use == REFUSED and content is False:
+        size = True
+    if content is None and size is None:
+        return None
+    declared, editable = declared_checks(entry)
+    evidence = ids(*(run for run in (none, present, zeros, half) if run))
+    said = sorted(declared)
+    shown = []
+    problems = []
+    if content is not None:
+        shown.append("checks the content" if content else "does not check the content")
+        if content != bool(declared & set(CONTENT_CHECKS)):
+            problems.append("a same-size file of zeros is " + ("refused" if content else "run")
+                            + f", the profile declares validation {said}")
+    missing_size = size is True and "size" not in declared
+    if size is not None:
+        shown.append("checks the size" if size else "does not check the size")
+        if size is False and "size" in declared:
+            problems.append(f"the first half of {name} is run, the profile declares a size check")
+    statement = f"the core {' and '.join(shown)} of {name}"
+    if problems:
+        return Verdict("validation_contradicted", "validation", said,
+                       f"{statement}: {'; '.join(problems)}", evidence)
+    if missing_size:
+        if not editable:
+            return Verdict("validation_contradicted", "validation", said,
+                           f"{statement}: the per-build validation has no size check", evidence)
+        value = [*(entry.get("validation") or []), "size"]
+        return Verdict("validation_size_added", "validation", value,
+                       f"{statement}: half of the file is refused while zeros of its size are run",
+                       evidence, True)
+    return Verdict("validation_agrees", "validation", entry.get("validation") or [], statement,
+                   evidence, editable)
+
+
+def rule_region(dump: Run, present: Run, entry: dict, game_region: str | None) -> str:
+    """What the core did with another region's dump under this file's name."""
+    return (f"{dump.label} staged as {entry['name']} with a {game_region or 'region-less'} game: "
+            f"{dump.verdict}, {entry['name']} itself: {present.verdict}")
 
 
 def rule_path(entry: dict, declared: str, opened: Opened) -> Verdict | None:
@@ -184,20 +250,25 @@ def rule_never_opened(entry: dict, record_ids: Sequence[str], content: str) -> V
                    f"{content}", tuple(record_ids))
 
 
-def rule_variants(status: str, labels: Iterable[str], entry: dict, record_id: str) -> Verdict | None:
-    """Copies of one file that change what the core shows: a pack choice to review."""
-    if status != "differs":
+def rule_variants(runs: Iterable[Run], entry: dict) -> Verdict | None:
+    """Copies of one file whose verdicts differ: a pack choice to review."""
+    runs = list(runs)
+    if len({run.verdict for run in runs}) < 2:
         return None
-    return Verdict("variant_changes_output", "variants", sorted(labels),
-                   f"the copies of {entry['name']} change what the core shows",
-                   (record_id,))
+    shown = ", ".join(f"{run.label} {run.verdict}" for run in runs)
+    return Verdict("variant_changes_verdict", "variants", sorted(run.label for run in runs),
+                   f"the copies of {entry['name']} do not boot alike: {shown}", ids(*runs))
 
 
-def merge_evidence(existing: dict | None, verdict: Verdict) -> dict:
-    """The entry's evidence: map with the verdict's record ids added under its field."""
+def merge_evidence(existing: dict | None, verdict: Verdict,
+                   keep: object = lambda rid: True) -> dict:
+    """The entry's evidence: map with the verdict's record ids added under its
+    field. An id `keep` refuses is dropped from that field: a record from
+    before the oracle no longer proves anything."""
     out = {key: list(value) for key, value in (existing or {}).items()}
-    ids = out.setdefault(verdict.field, [])
+    ids_ = [rid for rid in out.get(verdict.field, []) if keep(rid)]
     for rid in verdict.evidence:
-        if rid not in ids:
-            ids.append(rid)
-    return {key: out[key] for key in sorted(out)}
+        if rid not in ids_:
+            ids_.append(rid)
+    out[verdict.field] = ids_
+    return {key: out[key] for key in sorted(out) if out[key]}