diff --git a/scripts/core_probe.py b/scripts/core_probe.py index d16082927..0986d7e8c 100644 --- a/scripts/core_probe.py +++ b/scripts/core_probe.py @@ -619,6 +619,17 @@ def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> d return staged +def host_paths_hidden(text: str, places: Sequence[tuple[str, str]]) -> str: + """A core log line with the probe's own directories named, not spelled. + + Cores print the paths they open. Those paths belong to the host that ran + the probe and would make two identical runs on two machines differ. + """ + for prefix, name in places: + text = text.replace(prefix, name) + return text + + def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None, png_dir: Path | None) -> dict: save_dir = scratch / "save" @@ -650,6 +661,12 @@ def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_ou if proc.returncode != 0 or not out_path.exists(): raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}") result = json.loads(out_path.read_text(encoding="utf-8")) + places = [(str(system_dir.resolve()), ""), (str(save_dir.resolve()), "")] + if args.content: + places.append((str(Path(args.content).resolve().parent), "")) + places.append((str(ROOT), "")) + for line in result.get("logs", []): + line["text"] = host_paths_hidden(line["text"], places) if strace: opened, probed = files_under(parse_strace(trace_path.read_text(encoding="utf-8", errors="replace")), system_dir) @@ -822,22 +839,41 @@ def variants_for(args: argparse.Namespace) -> list[Variant]: return out +def unsettled(variant: dict, observable: str) -> bool: + """True when a variant cannot vouch for one value of the observable. + + A repetition that did not load, a file changed during the run, or two + repetitions that disagree on this observable. A disagreement on another + observable stays in the variant's flags without voiding this one. + """ + reps = variant["repetitions"] + if any(not rep.get("loaded") for rep in reps): + return True + if any("changed during the run" in flag for flag in variant["flags"]): + return True + values = {json.dumps((rep.get("observables") or {}).get(observable), sort_keys=True) for rep in reps} + return len(values) > 1 + + def check_result(variants: list[dict], observable: str) -> dict: groups: dict[str, list[str]] = {} - failed = [v["label"] for v in variants if v["flags"]] + failed = [v["label"] for v in variants if unsettled(v, observable)] for variant in variants: value = (variant["repetitions"][0].get("observables") or {}).get(observable) groups.setdefault(json.dumps(value, sort_keys=True), []).append(variant["label"]) if failed: status = "inconclusive" - statement = (f"{len(failed)} of {len(variants)} variants carry flags " - f"({', '.join(failed)}); no comparison is drawn from them") + statement = (f"{observable} is not settled in {len(failed)} of {len(variants)} variants " + f"({', '.join(failed)}): no comparison is drawn") elif len(groups) == 1: status = "same" statement = f"{observable} is identical across all {len(variants)} variants" else: status = "differs" statement = f"{observable} takes {len(groups)} distinct values across {len(variants)} variants" + other = sorted({v["label"] for v in variants if v["flags"]} - set(failed)) + if other: + statement += f". Other flags, in {len(other)} variants, are listed with them" return { "status": status, "statement": statement, diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index d66dc2770..709f762e5 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -163,9 +163,9 @@ class ParsingTests(unittest.TestCase): class ResultTests(unittest.TestCase): @staticmethod - def variant(label: str, value: str, flags: list[str] | None = None) -> dict: - return {"label": label, "flags": flags or [], - "repetitions": [{"observables": {"video_sha1": value}}]} + def variant(label: str, value: str, loaded: bool = True) -> dict: + return {"label": label, "flags": [], + "repetitions": [{"loaded": loaded, "observables": {"video_sha1": value}}]} def test_same_differs_inconclusive(self) -> None: same = core_probe.check_result([self.variant("a", "1"), self.variant("b", "1")], "video_sha1") @@ -173,10 +173,24 @@ class ResultTests(unittest.TestCase): differs = core_probe.check_result([self.variant("a", "1"), self.variant("b", "2")], "video_sha1") self.assertEqual(differs["status"], "differs") self.assertEqual(len(differs["groups"]), 2) - flagged = core_probe.check_result([self.variant("a", "1", ["x"]), self.variant("b", "1")], + flagged = core_probe.check_result([self.variant("a", "1", loaded=False), self.variant("b", "1")], "video_sha1") self.assertEqual(flagged["status"], "inconclusive") + def test_other_observable_does_not_void_the_check(self) -> None: + variant = {"label": "a", "flags": ["repetition 1 disagrees with repetition 0 on state_sha1"], + "repetitions": [{"loaded": True, "observables": {"video_sha1": "1", "state_sha1": "x"}}, + {"loaded": True, "observables": {"video_sha1": "1", "state_sha1": "y"}}]} + result = core_probe.check_result([variant], "video_sha1") + self.assertEqual(result["status"], "same") + self.assertIn("Other flags", result["statement"]) + self.assertEqual(core_probe.check_result([variant], "state_sha1")["status"], "inconclusive") + + def test_host_paths_hidden(self) -> None: + places = [("/tmp/x/system", ""), ("/tmp/x", "")] + self.assertEqual(core_probe.host_paths_hidden("/tmp/x/system/bios.bin NOT FOUND", places), + "/bios.bin NOT FOUND") + def test_timing_stats(self) -> None: stats = core_probe.timing_stats([1.0, 2.0, 3.0, 4.0, 10.0]) self.assertEqual((stats["median"], stats["mean"], stats["min"], stats["max"]), (3.0, 4.0, 1.0, 10.0)) diff --git a/wiki/tools.md b/wiki/tools.md index fdedf65d1..424628d3d 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -751,8 +751,11 @@ strace the record says that tracing was unavailable. Observables are `frame_sha1` (last frame), `video_sha1` (every frame, chained), `audio_sha1`, `state_sha1` (save state at the end), plus `state:SECTION.VAR` and `frame_hashes` when asked for. A repetition that disagrees with the first -one is flagged, and a `check` with a flagged variant reports `inconclusive` -instead of a comparison. A record carries the core build (sha256 of the +one is flagged. A `check` reports `inconclusive` instead of a comparison when, +in any variant, the compared observable differs between repetitions, a run did +not load, or a file changed during the run. A disagreement on another +observable stays flagged in the record. In core log lines, the staged system, +save and content directories read ``, `` and ``. A record carries the core build (sha256 of the `.so`, the compilers its `.comment` section names), the frontend commit, the host (CPU, kernel, Python, load before and after) and each input by hash. Core log lines arrive through the libretro log interface. Floating-point