feat: settle a probe check per observable

A check is inconclusive when the compared observable differs between repetitions, a run fails to load or a file changes during it. A disagreement on another observable stays flagged without voiding the comparison. Core log lines name the staged directories instead of the host paths.

scripts/core_probe.py

@@ -619,6 +619,17 @@ def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> d
     return staged
 
 
+def host_paths_hidden(text: str, places: Sequence[tuple[str, str]]) -> str:
+    """A core log line with the probe's own directories named, not spelled.
+
+    Cores print the paths they open. Those paths belong to the host that ran
+    the probe and would make two identical runs on two machines differ.
+    """
+    for prefix, name in places:
+        text = text.replace(prefix, name)
+    return text
+
+
 def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None,
              png_dir: Path | None) -> dict:
     save_dir = scratch / "save"
@@ -650,6 +661,12 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou
     if proc.returncode != 0 or not out_path.exists():
         raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}")
     result = json.loads(out_path.read_text(encoding="utf-8"))
+    places = [(str(system_dir.resolve()), "<system>"), (str(save_dir.resolve()), "<save>")]
+    if args.content:
+        places.append((str(Path(args.content).resolve().parent), "<content>"))
+    places.append((str(ROOT), "<repo>"))
+    for line in result.get("logs", []):
+        line["text"] = host_paths_hidden(line["text"], places)
     if strace:
         opened, probed = files_under(parse_strace(trace_path.read_text(encoding="utf-8", errors="replace")),
                                      system_dir)
@@ -822,22 +839,41 @@ def variants_for(args: argparse.Namespace) -> list[Variant]:
     return out
 
 
+def unsettled(variant: dict, observable: str) -> bool:
+    """True when a variant cannot vouch for one value of the observable.
+
+    A repetition that did not load, a file changed during the run, or two
+    repetitions that disagree on this observable. A disagreement on another
+    observable stays in the variant's flags without voiding this one.
+    """
+    reps = variant["repetitions"]
+    if any(not rep.get("loaded") for rep in reps):
+        return True
+    if any("changed during the run" in flag for flag in variant["flags"]):
+        return True
+    values = {json.dumps((rep.get("observables") or {}).get(observable), sort_keys=True) for rep in reps}
+    return len(values) > 1
+
+
 def check_result(variants: list[dict], observable: str) -> dict:
     groups: dict[str, list[str]] = {}
-    failed = [v["label"] for v in variants if v["flags"]]
+    failed = [v["label"] for v in variants if unsettled(v, observable)]
     for variant in variants:
         value = (variant["repetitions"][0].get("observables") or {}).get(observable)
         groups.setdefault(json.dumps(value, sort_keys=True), []).append(variant["label"])
     if failed:
         status = "inconclusive"
-        statement = (f"{len(failed)} of {len(variants)} variants carry flags "
-                     f"({', '.join(failed)}); no comparison is drawn from them")
+        statement = (f"{observable} is not settled in {len(failed)} of {len(variants)} variants "
+                     f"({', '.join(failed)}): no comparison is drawn")
     elif len(groups) == 1:
         status = "same"
         statement = f"{observable} is identical across all {len(variants)} variants"
     else:
         status = "differs"
         statement = f"{observable} takes {len(groups)} distinct values across {len(variants)} variants"
+    other = sorted({v["label"] for v in variants if v["flags"]} - set(failed))
+    if other:
+        statement += f". Other flags, in {len(other)} variants, are listed with them"
     return {
         "status": status,
         "statement": statement,

tests/test_core_probe.py

@@ -163,9 +163,9 @@ class ParsingTests(unittest.TestCase):
 
 class ResultTests(unittest.TestCase):
     @staticmethod
-    def variant(label: str, value: str, flags: list[str] | None = None) -> dict:
-        return {"label": label, "flags": flags or [],
-                "repetitions": [{"observables": {"video_sha1": value}}]}
+    def variant(label: str, value: str, loaded: bool = True) -> dict:
+        return {"label": label, "flags": [],
+                "repetitions": [{"loaded": loaded, "observables": {"video_sha1": value}}]}
 
     def test_same_differs_inconclusive(self) -> None:
         same = core_probe.check_result([self.variant("a", "1"), self.variant("b", "1")], "video_sha1")
@@ -173,10 +173,24 @@ class ResultTests(unittest.TestCase):
         differs = core_probe.check_result([self.variant("a", "1"), self.variant("b", "2")], "video_sha1")
         self.assertEqual(differs["status"], "differs")
         self.assertEqual(len(differs["groups"]), 2)
-        flagged = core_probe.check_result([self.variant("a", "1", ["x"]), self.variant("b", "1")],
+        flagged = core_probe.check_result([self.variant("a", "1", loaded=False), self.variant("b", "1")],
                                           "video_sha1")
         self.assertEqual(flagged["status"], "inconclusive")
 
+    def test_other_observable_does_not_void_the_check(self) -> None:
+        variant = {"label": "a", "flags": ["repetition 1 disagrees with repetition 0 on state_sha1"],
+                   "repetitions": [{"loaded": True, "observables": {"video_sha1": "1", "state_sha1": "x"}},
+                                   {"loaded": True, "observables": {"video_sha1": "1", "state_sha1": "y"}}]}
+        result = core_probe.check_result([variant], "video_sha1")
+        self.assertEqual(result["status"], "same")
+        self.assertIn("Other flags", result["statement"])
+        self.assertEqual(core_probe.check_result([variant], "state_sha1")["status"], "inconclusive")
+
+    def test_host_paths_hidden(self) -> None:
+        places = [("/tmp/x/system", "<system>"), ("/tmp/x", "<repo>")]
+        self.assertEqual(core_probe.host_paths_hidden("/tmp/x/system/bios.bin NOT FOUND", places),
+                         "<system>/bios.bin NOT FOUND")
+
     def test_timing_stats(self) -> None:
         stats = core_probe.timing_stats([1.0, 2.0, 3.0, 4.0, 10.0])
         self.assertEqual((stats["median"], stats["mean"], stats["min"], stats["max"]), (3.0, 4.0, 1.0, 10.0))

wiki/tools.md

@@ -751,8 +751,11 @@ strace the record says that tracing was unavailable.
 Observables are `frame_sha1` (last frame), `video_sha1` (every frame, chained),
 `audio_sha1`, `state_sha1` (save state at the end), plus `state:SECTION.VAR`
 and `frame_hashes` when asked for. A repetition that disagrees with the first
-one is flagged, and a `check` with a flagged variant reports `inconclusive`
-instead of a comparison. A record carries the core build (sha256 of the
+one is flagged. A `check` reports `inconclusive` instead of a comparison when,
+in any variant, the compared observable differs between repetitions, a run did
+not load, or a file changed during the run. A disagreement on another
+observable stays flagged in the record. In core log lines, the staged system,
+save and content directories read `<system>`, `<save>` and `<content>`. A record carries the core build (sha256 of the
 `.so`, the compilers its `.comment` section names), the frontend commit, the
 host (CPU, kernel, Python, load before and after) and each input by hash.
 Core log lines arrive through the libretro log interface. Floating-point