From 70c3bbd8536cd30851c007d964561c8661df786d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:58:48 +0100 Subject: [PATCH 1/6] feat: settle a file only when the run uses it hle_fallback needs the file to change the run: Beetle PSX hashes every file of its folder, so opened is not used. PCSX-ReARMed hides the BIOS logo by default: its BIOS load line stands for the bios stage. A check runs again only when the oracle entries of its content and core change, and the rules run on every pass from the kept records. --- schemas/core-probe-record.schema.json | 9 ++++++- scripts/campaign.py | 36 ++++++++++++++++++++++----- scripts/campaign_rules.py | 17 ++++++++++--- scripts/core_probe.py | 14 ++++++++--- tests/campaign/oracle.yml | 9 ++++++- tests/test_campaign.py | 21 ++++++++++++++++ tests/test_core_probe.py | 10 +++++++- wiki/tools.md | 16 +++++++----- 8 files changed, 111 insertions(+), 21 deletions(-) diff --git a/schemas/core-probe-record.schema.json b/schemas/core-probe-record.schema.json index 97687ec90..11e3232e7 100644 --- a/schemas/core-probe-record.schema.json +++ b/schemas/core-probe-record.schema.json @@ -1004,11 +1004,18 @@ } } } + }, + "bios_patterns": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Log lines by which the core says it loaded the console's own BIOS: they add the bios stage when its screens are hidden." } } } ], - "description": "What the verdicts were judged against: the reference frames of this content and core, the core's error patterns and error screens, from the oracle file at its sha256. Null when the run had no oracle entry." + "description": "What the verdicts were judged against: the reference frames of this content and core, the core's error patterns, error screens and BIOS load patterns, from the oracle file at its sha256. Null when the run had no oracle entry." } } } diff --git a/scripts/campaign.py b/scripts/campaign.py index de699b58d..ae0cd8eb4 100644 --- a/scripts/campaign.py +++ b/scripts/campaign.py @@ -903,6 +903,7 @@ class Campaign: self.store = ContentStore(self) self.content = load_content(paths.content_file) self.oracle_sha = file_sha256(paths.oracle) if paths.oracle.is_file() else "" + self.oracle_data = core_probe.load_oracle(paths.oracle)[0] self.deadline = 0.0 self.current: Pass | None = None self.dirty_at_start: set[str] = set() @@ -1055,9 +1056,14 @@ class Campaign: def remember(self, emulator: str, key: str, result: ProbeResult, core: Core | None, digest: str) -> None: + judged = None + if result.record and result.record["inputs"].get("content"): + judged = [result.record["inputs"]["content"]["sha1"], result.record["core"]["library_name"]] entry = {"status": result.status, "at": now_iso(), "profile_digest": digest, "core_sha256": core.sha256 if core else None, - "record_version": mr.RECORD_VERSION, "oracle_sha256": self.oracle_sha} + "record_version": mr.RECORD_VERSION, "oracle_sha256": self.oracle_digest(judged)} + if judged: + entry["oracle_key"] = judged if result.reason: entry["reason"] = result.reason if result.log: @@ -1073,9 +1079,27 @@ class Campaign: self.state["checks"].setdefault(emulator, {})[key] = { "status": PENDING, "reason": reason, "at": now_iso(), "profile_digest": digest} + def oracle_digest(self, key: list[str] | None) -> str: + """What a check's verdicts depend on in the oracle: the entries of its + content and core, else the whole file. Another core's references + never make a check run again.""" + if not key: + return self.oracle_sha + return mr.value_sha1(core_probe.oracle_for(self.oracle_data, *key)) + def needed(self, emulator: str, key: str, core: Core, digest: str) -> bool: - return should_run(self.state["checks"].get(emulator, {}).get(key), core.sha256, digest, - self.oracle_sha) + previous = self.state["checks"].get(emulator, {}).get(key) + judged = (previous or {}).get("oracle_key") + if previous and not judged and previous.get("record"): + record = self.read_record(emulator, previous["record"]) or {} + if (record.get("inputs") or {}).get("content"): + judged = [record["inputs"]["content"]["sha1"], record["core"]["library_name"]] + previous["oracle_key"] = judged + # Judged against the entries the record kept, not the file it named. + kept = {k: v for k, v in (record["inputs"].get("oracle") or {}).items() + if k not in ("file", "sha256")} + previous["oracle_sha256"] = mr.value_sha1({"bios_patterns": [], **kept}) + return should_run(previous, core.sha256, digest, self.oracle_digest(judged)) def check(self, emulator: str, key: str, core: Core, digest: str, argv: list[str], log_name: str) -> dict | None: @@ -1134,8 +1158,8 @@ class Campaign: keys[f"negative:{dest}"] = self.check_key(core, content, f"negative:{dest}") if self.settings.bench: keys["bench"] = self.check_key(core, content, "bench") - if not any(self.needed(emulator, key, core, digest) for key in keys.values()): - return + # The rules run on every pass, from the records kept when nothing + # needs to run again: a change of the table applies without a run. try: folder = self.store.get(content) load = load_file(folder, content.load) @@ -1174,7 +1198,7 @@ class Campaign: for label in labels[3:]: self.log(f"{emulator}: {rules.rule_region(rules.run_of(record, label), present, entry, content.region)}") if dest in self.opened(files_record, 0) or dest in self.opened(record, 0): - verdicts = rules.rule_need(none, present, entry) + verdicts = rules.rule_need(none, present, entry, rules.read(none, present, zeros, half)) validation = rules.rule_validation(zeros, half, none, present, entry) if validation: verdicts.append(validation) diff --git a/scripts/campaign_rules.py b/scripts/campaign_rules.py index ff1ec0d4b..dc21258c3 100644 --- a/scripts/campaign_rules.py +++ b/scripts/campaign_rules.py @@ -11,7 +11,7 @@ BIOS at all, `present` with the file alone, and wrong files staged under its name (zeros of its size, its first half, another region's dump). runs field settled - none boots hle_fallback true + none boots, the file changes the run hle_fallback true none fails, present boots required true hle_fallback false zeros refused, present boots validation a content check @@ -123,9 +123,20 @@ def agrees(entry: dict, verdict: Verdict) -> bool: return current(entry, verdict.field) == verdict.value -def rule_need(none: Run, present: Run, entry: dict) -> list[Verdict]: - """No BIOS at all against the file alone: required and hle_fallback.""" +def read(none: Run, *runs: Run) -> bool: + """True when the file changes the run: alone or as a wrong file, one + variant ends otherwise than the run with no BIOS, or reaches other + stages. A core can open a file without using it for this content (a + scan that hashes every file of the folder).""" + return any(run.verdict != none.verdict or run.stages != none.stages for run in runs if run) + + +def rule_need(none: Run, present: Run, entry: dict, used: bool = True) -> list[Verdict]: + """No BIOS at all against the file alone: required and hle_fallback. A + file the content never uses settles nothing.""" name = entry["name"] + if none.verdict == BOOTS and not used: + return [] if none.verdict == BOOTS: out = [Verdict("boots_without_file", "hle_fallback", True, f"the core boots the content with no BIOS at all, {name} included", diff --git a/scripts/core_probe.py b/scripts/core_probe.py index 0c6e91f45..bb35bf43e 100644 --- a/scripts/core_probe.py +++ b/scripts/core_probe.py @@ -1173,8 +1173,9 @@ DEFAULT_ORACLE = ROOT / "tests" / "campaign" / "oracle.yml" def oracle_for(data: dict, content_sha1: str | None, library_name: str | None) -> dict: """The oracle entries that judge one content on one core: its reference - frames, the error patterns every core shares and the core's own, and the - core's known error screens.""" + frames, the error patterns every core shares and the core's own, the + core's known error screens, and the log lines by which it says it loaded + the console's own BIOS.""" core = (data.get("cores") or {}).get(library_name or "") or {} references: list[dict] = [] for item in data.get("content") or (): @@ -1185,7 +1186,9 @@ def oracle_for(data: dict, content_sha1: str | None, library_name: str | None) - patterns = [p["pattern"] for p in (data.get("error_patterns") or ())] patterns += [p["pattern"] for p in core.get("error_patterns") or ()] screens = [{"sha1": s["sha1"], "shows": s["shows"]} for s in core.get("error_screens") or ()] - return {"references": references, "error_patterns": patterns, "error_screens": screens} + loaded = [p["pattern"] for p in core.get("bios_patterns") or ()] + return {"references": references, "error_patterns": patterns, "error_screens": screens, + "bios_patterns": loaded} def sampled(rep: dict) -> list[tuple[int, str]]: @@ -1218,6 +1221,11 @@ def judge(rep: dict, oracle: dict) -> tuple[str, dict]: references = {r["sha1"]: r for r in oracle.get("references") or ()} reached = [(frame, digest) for frame, digest in frames if digest in references] stages = list(dict.fromkeys(references[d]["stage"] for _, d in reached)) + # A core that hides the BIOS screens by default still says it loaded one. + if "bios" not in stages and any(re.search(pattern, line["text"]) + for pattern in oracle.get("bios_patterns") or () + for line in rep.get("logs") or ()): + stages.insert(0, "bios") game = [(frame, digest) for frame, digest in reached if references[digest]["stage"] == "game"] if game: frame, digest = game[0] diff --git a/tests/campaign/oracle.yml b/tests/campaign/oracle.yml index fd316c5ea..c027bfc7b 100644 --- a/tests/campaign/oracle.yml +++ b/tests/campaign/oracle.yml @@ -14,6 +14,9 @@ # cores:. A line a core logs and then carries on (a fallback to a # built-in BIOS) is not an error pattern. # error_screens per core, the sha1 of a frame that shows its BIOS error message. +# bios_patterns per core, log lines by which it says it loaded the console's +# own BIOS: they stand for the bios-stage screens of a core +# that hides them by default. # # A run that reaches no reference frame, matches no pattern or screen and # does not end on one colour is `unknown`: frames that differ from another @@ -47,7 +50,11 @@ cores: Without a BIOS, or with one of the wrong size, it logs "Using HLE BIOS" and runs its own (frontend/libretro.c:3893-3899 at c8816799). The BIOS boot logo is off by default (pcsx_rearmed_show_bios_bootlogo), so its - runs reach no bios-stage screen. + runs reach no bios-stage screen: the line it logs once a file passes + its signature check and is kept stands for them. + bios_patterns: + - pattern: '^found (US|EU|JP) BIOS file, crc32 ' + source_ref: frontend/libretro.c:3646-3669 at c8816799 error_patterns: [] error_screens: [] "PCSX1": diff --git a/tests/test_campaign.py b/tests/test_campaign.py index f981cf3ed..ec5753227 100644 --- a/tests/test_campaign.py +++ b/tests/test_campaign.py @@ -155,6 +155,14 @@ class RuleTable(unittest.TestCase): optional = rules.rule_need(NONE_BOOTS, PRESENT, {"name": "a", "required": False}) self.assertEqual([v.field for v in optional], ["hle_fallback"]) + def test_a_file_the_content_never_uses_settles_nothing(self) -> None: + unused = run(rules.BOOTS, "bios.bin", ("game",)) + self.assertFalse(rules.read(NONE_BOOTS, unused, unused, unused)) + self.assertEqual(rules.rule_need(NONE_BOOTS, unused, self.ENTRY, used=False), []) + # A wrong file that hangs where no file boots: the core reads it. + self.assertTrue(rules.read(NONE_BOOTS, unused, run(rules.HANG), unused)) + self.assertTrue(rules.read(NONE_BOOTS, PRESENT, None, None)) + def test_fails_without_and_boots_alone_settles_required(self) -> None: for failure in rules.FAILS: verdicts = rules.rule_need(run(failure, "none"), PRESENT, self.ENTRY) @@ -397,6 +405,19 @@ class StateAndResume(Workspace): self.assertTrue(campaign.should_run({**done, "status": "pending"}, "a", "d")) self.assertFalse(campaign.should_run({**done, "status": "failed"}, "a", "d")) + def test_only_its_own_oracle_entries_make_a_check_run_again(self) -> None: + key = [hashlib.sha1(b"GAME").hexdigest(), "Fake"] + before = self.make().oracle_digest(key) + data = yaml.safe_load(self.oracle.read_text()) + data["content"][0]["references"]["Other"] = {"recorded_from": "2026-10-27-000000000000", + "frames": [{"sha1": "a" * 40, "stage": "game", + "shows": "x"}]} + self.oracle.write_text(yaml.safe_dump(data), encoding="utf-8") + self.assertEqual(self.make().oracle_digest(key), before) + data["cores"] = {"Fake": {"error_patterns": [{"pattern": "BIOS", "reason": "r"}]}} + self.oracle.write_text(yaml.safe_dump(data), encoding="utf-8") + self.assertNotEqual(self.make().oracle_digest(key), before) + def test_evidence_does_not_change_the_digest(self) -> None: profile = yaml.safe_load(PROFILE) before = campaign.profile_digest(profile) diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index 5fb9c441b..e927cbe2d 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -211,6 +211,13 @@ class OracleTests(unittest.TestCase): self.assertEqual((evidence["rule"], evidence["frame"], evidence["stages"]), ("reference_frame", 20, ["bios", "game"])) + def test_a_bios_load_line_stands_for_hidden_bios_screens(self) -> None: + oracle = {**self.ORACLE, "bios_patterns": [r"^found US BIOS file"]} + verdict, evidence = core_probe.judge( + self.rep(frames((0, self.TITLE)), logs=["found US BIOS file, crc32 1: x"]), oracle) + self.assertEqual((verdict, evidence["stages"]), ("boots", ["bios", "game"])) + self.assertEqual(core_probe.judge(self.rep(frames((0, self.TITLE))), oracle)[1]["stages"], ["game"]) + def test_the_bios_screens_alone_are_not_a_boot(self) -> None: # A console that shows its logo and then refuses the disc never reached the game. verdict, evidence = core_probe.judge( @@ -275,6 +282,7 @@ class OracleTests(unittest.TestCase): picked = core_probe.oracle_for(data, "a" * 40, "Fake") self.assertEqual(picked["references"], [{"sha1": self.TITLE, "stage": "game", "shows": "t"}]) self.assertEqual(picked["error_patterns"], ["shared", "own"]) + self.assertEqual(picked["bios_patterns"], []) self.assertEqual(core_probe.oracle_for(data, "b" * 40, "Fake")["references"], []) self.assertEqual(core_probe.oracle_for(data, "a" * 40, "Nobody")["error_patterns"], ["shared"]) @@ -283,7 +291,7 @@ class OracleTests(unittest.TestCase): data = yaml.safe_load(core_probe.DEFAULT_ORACLE.read_text(encoding="utf-8")) patterns = list(data.get("error_patterns") or ()) for core in (data.get("cores") or {}).values(): - patterns += core.get("error_patterns") or [] + patterns += (core.get("error_patterns") or []) + (core.get("bios_patterns") or []) for screen in core.get("error_screens") or (): self.assertRegex(screen["sha1"], r"^[0-9a-f]{40}$") for pattern in patterns: diff --git a/wiki/tools.md b/wiki/tools.md index 3fe00c65a..622c37414 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -891,7 +891,9 @@ content by the sha1 of the file handed to the core and a core by its library name. A reference frame is recorded once with the full BIOS set and reviewed by looking at its PNG. An error pattern is a log line after which the core stops, read from its source: a line a core logs before it falls back to a -built-in BIOS is not one. Frames that differ from another run never make a +built-in BIOS is not one. A BIOS load pattern is the line a core logs once +it keeps the console's BIOS: it adds the `bios` stage for a core that hides +the BIOS screens by default (PCSX-ReARMed). Frames that differ from another run never make a boot. | Option | Effect | @@ -1000,7 +1002,7 @@ python scripts/campaign.py install-timer # opt-in: writes the autorun units, | Verdicts | Field | Written | |----------|-------|---------| -| `none` boots | `hle_fallback` | `true` | +| `none` boots, and the file changes the run: alone or as a wrong file, another verdict or other stages than with no BIOS | `hle_fallback` | `true`. A file the game never uses settles nothing: a core can open every file of the folder to hash it | | `none` fails (`bios_error`, `crash`, `hang`) and the file alone boots | `required`, `hle_fallback` | `true`, `false` | | Zeros of the file's size refused while the file boots | `validation` | a content check is proven. Agreeing adds evidence, a profile with no md5, sha1, sha256, crc32 or adler32 is an open finding | | Zeros used while the file boots | `validation` | no content check. A profile that declares one is an open finding | @@ -1026,16 +1028,18 @@ prints the record id with the screens that stay on for 3 samples or more in every repetition. A reviewer opens those PNGs and writes the chosen frames into `tests/campaign/oracle.yml` with their stage and what they show. -A run that agrees with the profile adds its record id to `evidence:` and +The rules run on every pass, from the kept records when no check needs to run +again, so a change of the table applies without a run. A run that agrees with +the profile adds its record id to `evidence:` and resolves an open finding of the same field. The profile is edited as text and the edit is refused unless the result parses to the same profile with only those fields changed. State lives in `.cache/campaign/state.json` of the main checkout: each check is `done`, `failed` (with the reason and its log) or `pending` (with the -reason), with the core sha256, the profile digest, the oracle sha256 and the -record version it ran against. A pass skips a check that is done or failed -against the same four, so it +reason), with the core sha256, the profile digest, the oracle entries of its +content and core (by sha1) and the record version it ran against. A pass +skips a check that is done or failed against the same four, so it takes new nightlies, changed profiles, newly fetched content and pending checks. The campaign's own `evidence:` writes do not count as a profile change. From e7d0c0183c92da2359b4203b76d873950c03c082 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:40:16 +0100 Subject: [PATCH 2/6] feat: show each test as one row with its proof The Tested block opens on one summary line, then a row per test: a short title, a result chip, a dot per variant and the date. Method, claim, computer, core build and the record wait in the row's details. Tests from before the boot oracle fold under one line, benchmarks get their own table. beetle_saturn shows under 120 words before anything is opened. --- scripts/sitemeasurements.py | 310 +++++++++++++++++++++++++++---- tests/test_measurements.py | 97 +++++++++- website/src/styles/retrobios.css | 75 ++++++++ wiki/measurements.md | 35 ++-- 4 files changed, 449 insertions(+), 68 deletions(-) diff --git a/scripts/sitemeasurements.py b/scripts/sitemeasurements.py index cc35ef362..4d065b631 100644 --- a/scripts/sitemeasurements.py +++ b/scripts/sitemeasurements.py @@ -18,6 +18,7 @@ its line at the last commit that touched measurements/. from __future__ import annotations +import html import json import os import re @@ -146,9 +147,20 @@ def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict, "runs": runs, "record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line), "sqlite": f"downloads/measurements/{entry.emulator}.sqlite", + "host_short": short_host(record["host"]), + "groups": variant_groups(record), + "proves": proves(record["id"], profile), + "timing": ({"ms_per_frame": record["timing"]["ms_per_frame"]["median"], + "fps": record["timing"]["fps"]["median"], + "stdev": record["timing"]["ms_per_frame"]["stdev"]} + if record.get("timing") else None), } +def with_title(item: dict) -> dict: + return {**item, "title": short_title(item)} + + def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict, system_names: dict[str, str] | None = None) -> dict: """One item per (emulator, subject): its latest record not superseded.""" @@ -161,7 +173,8 @@ def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict, for key in sorted(by_subject): entries = by_subject[key] latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line)) - items.append(index_item(latest, len(entries), source or {}, profiles, system_names or {})) + items.append(with_title(index_item(latest, len(entries), source or {}, profiles, + system_names or {}))) items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"])) return { "schema_version": INDEX_SCHEMA_VERSION, @@ -514,9 +527,14 @@ def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulato # -------------------------------------------------------------------------- def by_emulator(index: dict) -> dict[str, list[dict]]: + """Each emulator's items, with the count of its findings still open.""" + opened: dict[str, int] = {} + for line in (index.get("campaign") or {}).get("open") or (): + opened[line["emulator"]] = opened.get(line["emulator"], 0) + 1 out: dict[str, list[dict]] = {} for item in index.get("items", []): - out.setdefault(item["emulator"], []).append(item) + out.setdefault(item["emulator"], []).append( + {**item, "open_findings": opened.get(item["emulator"], 0)}) return out @@ -532,52 +550,262 @@ def _short(commit: str | None) -> str: return f"`{commit[:12]}`" if commit else "unknown" -def tested_item(item: dict) -> list[str]: - """One test: what ran, the result, the method and host beside it, the record.""" - title = item["claim"] or f"Benchmark of {item['content'] or 'the core alone'}" - runs = "1 run" if item["runs"] == 1 else f"{item['runs']} runs, the latest shown" - core = item["core"] - build = f"{_text(core['library'])} at {_short(core['commit'])}" if core.get("commit") \ - else _text(core["library"]) - lines = [f"- **{_text(title)}**", ""] - if len(item.get("verdicts") or ()) > 1: - lines += [f" **Result:** `{item['status']}`.", ""] - lines += [f" - {_text(v['variant'])}: `{v['verdict']}`. {_text(v['detail'])}." - for v in item["verdicts"]] - lines.append("") +# A verdict or a comparison, as the block says it: words and a chip kind. +RESULT_WORDS = { + "boots": ("works", "success"), + "bios_error": ("BIOS error", "danger"), + "crash": ("crash", "danger"), + "hang": ("hang", "warning"), + "unknown": ("unknown", "muted"), + "mixed": ("differs", "info"), + "same": ("no difference", "muted"), + "differs": ("differs", "info"), + "inconclusive": ("inconclusive", "warning"), + "no verdict": ("no verdict", "muted"), + "ran": ("ran", "muted"), +} +# Words a subject slug spells that read as one term. +TERMS = {"bios": "BIOS", "vcd": "Video CD", "cd": "CD", "jp": "JP", "us": "US", "eu": "EU", + "cr2": "CR2", "mpeg": "MPEG", "psx": "PSX", "hle": "HLE", "usa": "USA"} +# The campaign's own check names, as the block says them. +CHECK_NAMES = {"files": "full set", "none": "no BIOS", "negative": "wrong files", + "references": "reference run", "variants": "copies", "alias": "other name", + "path": "other path", "bench": "benchmark"} +TITLE_AXIS_WORDS = 4 + + +def _words(text: str) -> list[str]: + return re.findall(r"[a-z0-9]+", str(text or "").lower()) + + +def short_content(label: str | None) -> str: + """A content label cut to the game's name: no source prefix, no + edition, region or identity detail.""" + text = str(label or "").strip() + if ":" in text.split("(")[0]: + text = text.split(":", 1)[1].strip() + for cut in (" - ", " (", ", "): + text = text.split(cut, 1)[0] + return text.strip() + + +def short_title(item: dict) -> str: + """A test's name in a few words: the file it varies, the game, and what + the subject adds to them (a build, a region, a check of the campaign).""" + thing = Path(str(item.get("variable") or "")).name + game = short_content(item.get("content")) + known = set(_words(thing)) | set(_words(item.get("content"))) | set(_words(game)) + known |= set(_words(Path(str((item.get("core") or {}).get("file") or "")).name.split("_libretro")[0])) + axis = [] + for word in _words(item.get("subject")): + if word in known or re.fullmatch(r"[0-9a-f]{8}", word): + continue + issue = re.fullmatch(r"issue(\d+)", word) + word = f"issue {issue[1]}" if issue else CHECK_NAMES.get(word, TERMS.get(word, word)) + if word not in axis: + axis.append(word) + if len(axis) > TITLE_AXIS_WORDS: + axis = [*axis[:TITLE_AXIS_WORDS], "..."] + head = ", ".join(part for part in (f"`{thing}`" if thing else "", game) if part) + title = f"{head} ({' '.join(axis)})" if axis and head else head or " ".join(axis) + return title or str(item.get("subject") or "") + + +def short_host(host: dict) -> str: + """The computer as a reader names it: the CPU model and the OS.""" + cpu = str(host.get("cpu_model") or "unknown CPU") + cpu = re.sub(r"\(R\)|\(TM\)|\bCPU\b|@.*$|^\d+(?:st|nd|rd|th) Gen ", "", cpu) + cpu = " ".join(cpu.split()) + system = str(host.get("os") or "") + name = "Linux" if "Linux" in system else system.split("-")[0] or "unknown OS" + if "microsoft" in system.lower(): + name += " (WSL2)" + return f"{cpu}, {name}" + + +def variant_groups(record: dict) -> list[dict]: + """One entry per variant: the result group it falls in, and whether it + staged no file. A run that carries verdicts is grouped by verdict, a + comparison by the value its observable took.""" + check = record.get("check") or {} + position: dict[str, int] = {} + result: dict[str, str] = {} + if all("verdict" in v for v in record["variants"]) and ( + record["kind"] == "run" or check.get("observable") == "frame_hashes"): + for variant in record["variants"]: + position.setdefault(variant["verdict"], len(position)) + result[variant["label"]] = RESULT_WORDS[variant["verdict"]][0] + groups = {v["label"]: position[v["verdict"]] for v in record["variants"]} else: - lines += [f" **Result:** `{item['status']}`. {_text(item['statement'])}.", ""] - if item.get("method"): - lines += [f" **Method:** {_text(item['method'])}", ""] - lines += [ - f" **Ran on:** {_text(item['host'])}, {item['date'][:10]}. Core build: {build}. " - f"{item['variants']} variant{'s' if item['variants'] != 1 else ''}, " - f"{item['repetitions']} repetition{'s' if item['repetitions'] != 1 else ''} each" - + (f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" if item["flags"] else "") - + ".", - "", - f" [Full record]({item['record_url']}), {runs}.", - "", + groups = {} + for number, group in enumerate((check.get("result") or {}).get("groups") or ()): + for label in group["variants"]: + groups[label] = number + result[label] = f"result {chr(ord('A') + number % 26)}" + varied = check.get("variable") + return [{"variant": v["label"], "group": groups.get(v["label"], 0), + "result": result.get(v["label"], ""), + "absent": bool(varied) and varied in v["system_files"] and v["system_files"][varied] is None} + for v in record["variants"]] + + +def proves(record_id: str, profile: dict) -> list[str]: + """The profile fields this record is evidence for, as `field of file`.""" + out = [] + for entry in profile.get("files") or (): + for name, ids in sorted((entry.get("evidence") or {}).items()): + if record_id in ids: + out.append(f"{name} of {entry.get('name')}") + return out + + +# An MDX page reads a brace as the start of an expression, in text and in +# an attribute alike. +_BRACES = str.maketrans({"{": "{", "}": "}"}) + + +def _attr(value: object) -> str: + return html.escape(" ".join(str(value or "").split()), quote=True).translate(_BRACES) + + +def _cell(value: object) -> str: + """Record text placed in raw HTML: escaped, code spans kept, a semicolon + joining two clauses ending a sentence as on the rest of the site.""" + text = " ".join(str(value or "").split()) + text = re.sub(r";\s+(\w)", lambda m: ". " + m[1].upper(), text) + return markup.inline_html(text).translate(_BRACES) + + +def _dots(item: dict) -> str: + groups = item.get("groups") or [] + names = "; ".join(f"{g['variant']}: {g['result']}" for g in groups) + dots = "".join( + f'' for g in groups) + return (f'' + f"{dots}") + + +def _chip(item: dict) -> str: + words, kind = RESULT_WORDS.get(item["status"], (item["status"], "muted")) + return f'{html.escape(words)}' + + +def _sentence_html(label: str, value: object, markup_ready: bool = False) -> str: + """One line of the details, or nothing: an empty value is never printed.""" + text = (str(value or "") if markup_ready else _cell(value)).strip().rstrip(".") + return f"

{label}: {text}.

" if text else "" + + +def _details(item: dict) -> list[str]: + core = item["core"] + build = _cell(core["library"]) + (f" at {html.escape(core['commit'][:12])}" + if core.get("commit") else "") + reps = f"{item['variants']} variant{'s' if item['variants'] != 1 else ''}, " \ + f"{item['repetitions']} repetition{'s' if item['repetitions'] != 1 else ''} each" + if item["flags"]: + reps += f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" + runs = "1 run" if item["runs"] == 1 else f"{item['runs']} runs, the latest shown" + lines = [ + _sentence_html("Claim", item.get("claim")), + _sentence_html("Result", item.get("statement")), + _sentence_html("Method", item.get("method")), + _sentence_html("Proves", ", ".join(item.get("proves") or [])), + _sentence_html("Ran on", item["host"]), + _sentence_html("Core build", build and f"{build}. {reps}", markup_ready=True), + f'

Full record, {runs}.

', + ] + return [line for line in lines if line] + + +def tested_counts(items: list[dict]) -> dict: + tests = [i for i in items if i["kind"] != "bench"] + return {"tests": len(items), "confirmed": sum(1 for i in items if i.get("proves")), + "open": max((i.get("open_findings", 0) for i in items), default=0), + "benchmarks": len(items) - len(tests), + "last": max((i["date"][:10] for i in items), default="")} + + +def core_label(item: dict) -> str: + return Path(str(item["core"].get("file") or "")).name.split("_libretro")[0] + + +def _row(item: dict, page_host: str, main_core: str) -> list[str]: + elsewhere = (item.get("host_short") or item["host"]) != page_host + title = _cell(item.get("title") or short_title(item)) + if core_label(item) != main_core: + title += f' {html.escape(core_label(item))}' + return [ + '
', + ("" + f'{title}' + f'{_chip(item)}' + f'{_dots(item)}' + f'{item["date"][:10]}' + + (f' {_attr(item.get("host_short"))}' if elsewhere else "") + + ""), + '
', + *_details(item), + "
", + "
", ] - return lines def tested_section(name: str, items: list[dict]) -> list[str]: - """The Tested block of an emulator page, empty when nothing was recorded.""" + """The Tested block of an emulator page, empty when nothing was recorded. + + The answer first: one summary line, then one row per test with its + result chip, a dot per variant and its date. What proves it (method, + claim, computer, core build, record) waits in each row's details. Tests + recorded before the boot oracle fold under one line, and benchmarks get + their own table at the end. + """ if not items: return [] - lines = [ - "## Tested", - "", - f"This emulator was run in real conditions by the [core probe](../{METHOD_PAGE}). " - f"Each test below is the latest of its kind. " - f"[All measurements](../{EXPLORER_PAGE}), " - f"[these records as SQLite](../downloads/measurements/{name}.sqlite).", - "", - ] - for item in sorted(items, key=lambda i: (i["date"], i["subject"]), reverse=True): - lines += tested_item(item) - return lines + counts = tested_counts(items) + hosts = [i.get("host_short") or i["host"] for i in items] + page_host = max(sorted(set(hosts)), key=hosts.count) + cores = [core_label(i) for i in items] + main_core = max(sorted(set(cores)), key=cores.count) + tests = f"{counts['tests']} test{'s' if counts['tests'] != 1 else ''}" + summary = (f"{tests} · {counts['confirmed']} confirmed · {counts['open']} open · " + f"last run {counts['last']} on {page_host}.") + links = (f'How · ' + f'All tests · ' + f'SQLite') + ordered = sorted(items, key=lambda i: (i["date"], i["subject"], core_label(i)), reverse=True) + benches = [i for i in ordered if i["kind"] == "bench"] + earlier = [i for i in ordered if i["kind"] != "bench" and i["status"] == "no verdict"] + current = [i for i in ordered if i["kind"] != "bench" and i["status"] != "no verdict"] + lines = ["## Tested", "", '
', + f'

{html.escape(summary)} {links}

'] + if current or earlier: + lines.append('") + for item in current: + lines += _row(item, page_host, main_core) + if earlier: + lines += ['
', + f"{len(earlier)} earlier test{'s' if len(earlier) != 1 else ''}, " + "before the boot oracle"] + for item in earlier: + lines += _row(item, page_host, main_core) + lines.append("
") + if benches: + # One line: MDX reads a table split over lines as paragraphs. + table = ['' + ""] + for item in benches: + timing = item.get("timing") or {} + title = _cell(item.get("title") or short_title(item)) + if core_label(item) != main_core: + title += f' {html.escape(core_label(item))}' + table.append( + f'' + f'' + f'') + lines.append("".join([*table, "
Benchmarkms per framefpsstdevruns
{title}{timing.get("ms_per_frame", 0):.3f}{timing.get("fps", 0):.1f}{timing.get("stdev", 0):.3f}{item["repetitions"]}
"])) + return [*lines, "
", ""] def hosts_table(index: dict) -> list[str]: diff --git a/tests/test_measurements.py b/tests/test_measurements.py index c7eb2536e..5bf12a97b 100644 --- a/tests/test_measurements.py +++ b/tests/test_measurements.py @@ -220,7 +220,8 @@ class ReadAndIndex(Workspace): keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind", "claim", "method", "observable", "variable", "status", "statement", "content", "variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id", - "runs", "record_url", "sqlite", "verdicts"} + "runs", "record_url", "sqlite", "verdicts", "title", "host_short", "groups", + "proves", "timing"} for item in index["items"]: self.assertEqual(set(item), keys) alpha = {i["subject"]: i for i in index["items"] if i["emulator"] == "alpha"} @@ -247,11 +248,9 @@ class ReadAndIndex(Workspace): self.assertNotIn("differ", statement) # A check on another observable keeps its comparison. self.assertEqual(sm.result_of(full_record("alpha", "c", "2026-10-06T10:00:00+00:00"))[0], "differs") - item = {"claim": "c", "runs": 1, "core": {"library": "Fake 1", "commit": None}, "status": "mixed", - "statement": "", "verdicts": sm.verdicts_of(record), "method": "m", "host": "h", - "date": "2026-10-06", "variants": 2, "repetitions": 2, "flags": 0, "record_url": "u"} - text = "\n".join(sm.tested_item(item)) - self.assertIn("v1: `hang`", text) + groups = sm.variant_groups(record) + self.assertEqual([(g["variant"], g["result"], g["group"]) for g in groups], + [("v0", "works", 0), ("v1", "hang", 1)]) def test_links_name_the_last_commit_that_touched_the_records(self) -> None: first = self.repository() @@ -431,12 +430,90 @@ class SitePages(unittest.TestCase): self.assertEqual(sm.tested_section("beetle_saturn", []), []) text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM])) self.assertTrue(text.startswith("## Tested\n")) - for expected in ("**Result:** `same`", "**Method:** Twice per card. Every input fixed.", "**Ran on:** Test CPU, 12 CPUs", - self.ITEM["record_url"], "1 flag raised", "2 runs", - "../downloads/measurements/beetle_saturn.sqlite", "../wiki/measurements.md", - "<image>"): + for expected in ("no difference", "Method: Twice per card. Every input fixed.", + "Ran on: Test CPU, 12 CPUs", self.ITEM["record_url"], + "1 flag raised", "2 runs", "downloads/measurements/beetle_saturn.sqlite", + "wiki/measurements/", "<image>", "1 test · 0 confirmed · 0 open"): self.assertIn(expected, text) + def item(self, **values: object) -> dict: + return {**self.ITEM, "host_short": "Test CPU, Linux", **values} + + def test_summary_counts(self) -> None: + items = [self.item(proves=["hle_fallback of a.bin"], open_findings=2), + self.item(subject="b", date="2026-10-28T00:00:00+00:00", open_findings=2), + self.item(kind="bench", status="measured", subject="c", + timing={"ms_per_frame": 1.5, "fps": 660.0, "stdev": 0.01})] + self.assertEqual(sm.tested_counts(items), {"tests": 3, "confirmed": 1, "open": 2, "benchmarks": 1, + "last": "2026-10-28"}) + text = "\n".join(sm.tested_section("beetle_saturn", items)) + self.assertIn("3 tests · 1 confirmed · 2 open · last run 2026-10-28 on Test CPU, Linux.", text) + self.assertIn("1.500660.0", text) + + def test_short_titles(self) -> None: + cases = [ + ({"subject": "mpeg-card-lunar-cr2-build", "variable": "mpeg_card.bin", + "content": "Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00"}, + "`mpeg_card.bin`, Lunar (CR2 build)"), + ({"subject": "negative-scph5501-bin-mednafen-psx-bust-a-move-2-arcade-edition-usa", + "variable": "scph5501.bin", "content": "Bust-A-Move 2 - Arcade Edition (USA)", + "core": {"file": "mednafen_psx_libretro.so"}}, + "`scph5501.bin`, Bust-A-Move 2 (wrong files)"), + ({"subject": "issue85-lunar-opening-movie", "variable": None, + "content": "Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00"}, + "Lunar (issue 85 opening movie)"), + ({"subject": "with-command-auth-device-and-command-get-9a211715", "variable": "mpeg_card.bin", + "content": "Redump: Lunar"}, "`mpeg_card.bin`, Lunar (with command auth device ...)"), + ] + for item, title in cases: + self.assertEqual(sm.short_title(item), title) + self.assertEqual(sm.short_host({"cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K", + "os": "Linux-6.18.54.1-microsoft-standard-WSL2-x86_64"}), + "Intel Core i9-13900K, Linux (WSL2)") + + def test_no_empty_sentence(self) -> None: + text = "\n".join(sm.tested_section("beetle_saturn", [self.item(claim=None, method=None, statement="", + kind="run", status="ran")])) + self.assertNotRegex(text, r"\s*\.|:\s*\.\s|`ran`\. \.") + self.assertNotIn("Claim:", text) + self.assertNotIn("Method:", text) + + def test_dots_follow_the_result_groups(self) -> None: + record = full_record("alpha", "boot", "2026-10-06T10:00:00+00:00", values=("1" * 40, "2" * 40, "1" * 40)) + record["check"]["variable"] = "card.bin" + record["variants"][2]["system_files"] = {"card.bin": None} + groups = sm.variant_groups(record) + self.assertEqual([g["group"] for g in groups], + [i for i, group in enumerate(record["check"]["result"]["groups"]) + for _ in group["variants"]][:1] + [1, 0]) + self.assertEqual([g["absent"] for g in groups], [False, False, True]) + text = "\n".join(sm.tested_section("alpha", [self.item(groups=groups)])) + self.assertEqual(re.findall(r'rb-dot rb-dot-(\d)( rb-dot-empty)?"', text), + [("0", ""), ("1", ""), ("0", " rb-dot-empty")]) + self.assertIn('aria-label="3 variants. v0: result A; v1: result B; v2: result A"', text) + + def test_the_details_hold_the_method_and_the_host(self) -> None: + text = "\n".join(sm.tested_section("beetle_saturn", [self.item()])) + details = text.split('
', 1)[1].split("
", 1)[0] + self.assertIn("Twice per card", details) + self.assertIn("Test CPU, 12 CPUs", details) + self.assertIn(self.ITEM["record_url"], details) + + def test_the_beetle_saturn_block_stays_short(self) -> None: + stored = mr.read_records(REPO_ROOT) + profiles = sm.load_profiles(REPO_ROOT / "emulators") + index = sm.build_index(stored, {"repo": REPO_URL, "commit": COMMIT}, profiles) + items = sm.by_emulator(index).get("beetle_saturn") + if not items: + self.skipTest("no beetle_saturn record") + block = "\n".join(sm.tested_section("beetle_saturn", items)) + # What a reader sees before opening anything: the closed details hide their body. + shown = re.sub(r'
.*?
', " ", block, flags=re.S) + shown = re.sub(r'
.*?
\s*', " ", shown, + flags=re.S) + shown = re.sub(r"<[^>]+>", " ", shown) + self.assertLess(len(re.findall(r"[A-Za-z0-9][\w.'-]*", shown)), 120, shown) + def test_hosts_table_on_the_method_page(self) -> None: page = sm.with_hosts(f"x\n\n{sm.HOSTS_MARKER}\n", {"items": [self.ITEM, {**self.ITEM, "host": "Test CPU, 12 CPUs, load 1 before, 2 after"}]}) diff --git a/website/src/styles/retrobios.css b/website/src/styles/retrobios.css index 77831fc28..865b72e04 100644 --- a/website/src/styles/retrobios.css +++ b/website/src/styles/retrobios.css @@ -877,3 +877,78 @@ starlight-tabs [role='tab'] { white-space: nowrap; } @media (min-width: 50rem) { header .site-title img { height: calc(4rem - 2 * var(--sl-nav-pad-y)); width: auto; } } + +/* The Tested block of an emulator page (scripts/sitemeasurements.py): a + summary line, then one row per test, each a
whose summary is + the row and whose body is the proof. Rows stack into two lines on a + phone. A dot per variant, its group in colour and its name in a title + and the strip's label; a variant that staged no file is a ring. */ +.sl-markdown-content .rb-tested-summary { color: var(--rb-text-secondary); font-size: 0.9rem; } +.sl-markdown-content :is(.rb-tested-head, .rb-tested-row > summary) { + display: grid; + grid-template-columns: 1.25rem minmax(0, 1fr) 8.5rem 9rem 6rem; + gap: 0.5rem; + align-items: center; +} +.sl-markdown-content .rb-tested-head { + margin-top: 0.75rem; + padding: 0 0 0.25rem; + border-bottom: 1px solid var(--rb-border-strong); + color: var(--rb-text-secondary); + font-size: 0.8rem; + font-weight: 600; +} +.sl-markdown-content .rb-tested-head::before { content: ""; } +/* Starlight draws a side rule on every details: a row has a rule below only. */ +.sl-markdown-content details:is(.rb-tested-row, .rb-tested-earlier) { border-inline-start: 0; padding-inline-start: 0; } +.sl-markdown-content details.rb-tested-row { margin: 0; border-bottom: 1px solid var(--rb-border); } +.sl-markdown-content details.rb-tested-row:has(> summary:hover) { border-color: var(--rb-border); } +.sl-markdown-content .rb-tested-row > summary { + min-height: 2.25rem; + padding: 0.3rem 0; + cursor: pointer; + font-size: 0.9rem; +} +.sl-markdown-content .rb-tested-row > summary::before { background-color: var(--rb-text-secondary); margin: 0; } +.sl-markdown-content .rb-tested-row > summary:hover { background: var(--rb-surface); } +.sl-markdown-content .rb-tested-test { font-weight: 600; overflow-wrap: anywhere; } +.sl-markdown-content :is(.rb-tested-test, .rb-tested-date) small { color: var(--rb-text-secondary); font-weight: 400; } +.sl-markdown-content .rb-tested-date { color: var(--rb-text-secondary); font-variant-numeric: tabular-nums; } +.sl-markdown-content .rb-tested-more { + margin: 0 0 0.75rem 1.75rem; + padding: 0.5rem 0.75rem; + border-left: 2px solid var(--rb-border-strong); + font-size: 0.875rem; +} +.sl-markdown-content .rb-tested-more p { margin: 0.35rem 0 0; } +.sl-markdown-content .rb-tested-more p:first-child { margin-top: 0; } +.sl-markdown-content details.rb-tested-earlier { margin-top: 0.75rem; } +.sl-markdown-content details.rb-tested-earlier > summary { font-size: 0.9rem; color: var(--rb-text-secondary); cursor: pointer; } +.sl-markdown-content .rb-dots { display: inline-flex; flex-wrap: wrap; gap: 3px; } +.sl-markdown-content .rb-dot { + width: 10px; + height: 10px; + border-radius: 50%; + border: 2px solid var(--rb-dot); + background: var(--rb-dot); +} +.sl-markdown-content .rb-dot-empty { background: transparent; } +.sl-markdown-content .rb-dot-0 { --rb-dot: var(--rb-ok-fg); } +.sl-markdown-content .rb-dot-1 { --rb-dot: var(--rb-info-fg); } +.sl-markdown-content .rb-dot-2 { --rb-dot: var(--rb-warn-fg); } +.sl-markdown-content .rb-dot-3 { --rb-dot: var(--rb-err-fg); } +.sl-markdown-content .rb-dot-4 { --rb-dot: var(--rb-muted-fg); } +.sl-markdown-content table.rb-tested-bench { margin-top: 1rem; font-size: 0.875rem; } +.sl-markdown-content table.rb-tested-bench td:not(:first-child) { font-variant-numeric: tabular-nums; text-align: right; } +@media (max-width: 50rem) { + .sl-markdown-content div.rb-tested-head { display: none; } + .sl-markdown-content details.rb-tested-row > summary { + grid-template-columns: 1.25rem auto auto minmax(0, 1fr); + row-gap: 0.2rem; + } + .sl-markdown-content .rb-tested-row > summary::before { grid-row: 1; } + .sl-markdown-content .rb-tested-test { grid-column: 2 / -1; } + .sl-markdown-content .rb-tested-result { grid-column: 2; } + .sl-markdown-content .rb-tested-date { text-align: right; } + .sl-markdown-content .rb-tested-more { margin-left: 0.5rem; } +} diff --git a/wiki/measurements.md b/wiki/measurements.md index b251058d0..a39682963 100644 --- a/wiki/measurements.md +++ b/wiki/measurements.md @@ -14,34 +14,35 @@ the emulator's page shows a **Tested** block. Every result is on the ## What a test does - A script runs the emulator with no window and no sound. -- It starts a game and runs it for a set time. -- Each run starts fresh and runs at least twice. -- Each run gets one result, by comparing its screens with ones we checked by eye. +- It starts a game and runs it for a set time, at least twice. +- It compares the screens with ones we checked by eye. ## What a result says -- **boots**: the game reached a screen it shows when it works. -- **bios_error**: the emulator said a BIOS is missing or wrong. -- **crash**: the emulator stopped. -- **hang**: the screen stayed blank, or the run never ended. -- **unknown**: none of these could be shown. -- **measured**: a speed test, given as time per frame. -- **no verdict**: an older test that only compared screens. +| Result | Meaning | +|--------|---------| +| works | the game reached a screen it shows when it works | +| BIOS error | the emulator said a BIOS is missing or wrong | +| crash | the emulator stopped | +| hang | the screen stayed blank, or the run never ended | +| unknown | none of these could be shown | +| no verdict | an older test that only compared screens | +| ms per frame | a speed test | ## How a file is proven -The game runs with no BIOS, with the BIOS alone, and with a wrong file under -its name: zeros, half the file, or another region's BIOS. +The game runs with no BIOS and with the BIOS alone. Then it runs with a wrong +file under the BIOS name: zeros, half the file, or another region's BIOS. -- **Required**: no BIOS fails, the BIOS alone boots. -- **Built-in stand-in**: no BIOS still boots. +- **Required**: no BIOS fails, the BIOS alone works. +- **Built-in stand-in**: no BIOS still works. - **Checked**: a wrong file is refused. ## When tests run -A maintainer starts a test run. A clear result updates the emulator's page and -names the test as proof. Anything else waits on the -[Measurements](../measurements.md) page for someone to read the code. +A maintainer starts a test run. A clear result updates the emulator's page. +Anything else waits on the [Measurements](../measurements.md) page for someone +to read the code. ## What a result keeps From b6ed5ba0a1e877f063d3915c6b8f8d28e6bc6482 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:48:16 +0100 Subject: [PATCH 3/6] feat: group tested rows by the file they vary Rows sit under the file under test. A title names the game and what sets the test apart from the others of its file (build, clock, region, files tried), never cut. Zero counts leave the summary. A legend explains the dots. The chips say depends on the file and same with every file. Ran on shows the short host. --- scripts/sitemeasurements.py | 166 ++++++++++++++++++++++++++----- tests/test_measurements.py | 49 +++++++-- website/src/styles/retrobios.css | 13 ++- 3 files changed, 190 insertions(+), 38 deletions(-) diff --git a/scripts/sitemeasurements.py b/scripts/sitemeasurements.py index 4d065b631..9a915ed3c 100644 --- a/scripts/sitemeasurements.py +++ b/scripts/sitemeasurements.py @@ -148,6 +148,7 @@ def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict, "record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line), "sqlite": f"downloads/measurements/{entry.emulator}.sqlite", "host_short": short_host(record["host"]), + "setup": setup_of(record), "groups": variant_groups(record), "proves": proves(record["id"], profile), "timing": ({"ms_per_frame": record["timing"]["ms_per_frame"]["median"], @@ -557,9 +558,9 @@ RESULT_WORDS = { "crash": ("crash", "danger"), "hang": ("hang", "warning"), "unknown": ("unknown", "muted"), - "mixed": ("differs", "info"), - "same": ("no difference", "muted"), - "differs": ("differs", "info"), + "mixed": ("depends on the file", "info"), + "same": ("same with every file", "muted"), + "differs": ("depends on the file", "info"), "inconclusive": ("inconclusive", "warning"), "no verdict": ("no verdict", "muted"), "ran": ("ran", "muted"), @@ -570,8 +571,7 @@ TERMS = {"bios": "BIOS", "vcd": "Video CD", "cd": "CD", "jp": "JP", "us": "US", # The campaign's own check names, as the block says them. CHECK_NAMES = {"files": "full set", "none": "no BIOS", "negative": "wrong files", "references": "reference run", "variants": "copies", "alias": "other name", - "path": "other path", "bench": "benchmark"} -TITLE_AXIS_WORDS = 4 + "path": "other path", "bench": "benchmark", "boot": "boot test", "alone": "file alone"} def _words(text: str) -> list[str]: @@ -604,13 +604,102 @@ def short_title(item: dict) -> str: word = f"issue {issue[1]}" if issue else CHECK_NAMES.get(word, TERMS.get(word, word)) if word not in axis: axis.append(word) - if len(axis) > TITLE_AXIS_WORDS: - axis = [*axis[:TITLE_AXIS_WORDS], "..."] head = ", ".join(part for part in (f"`{thing}`" if thing else "", game) if part) title = f"{head} ({' '.join(axis)})" if axis and head else head or " ".join(axis) return title or str(item.get("subject") or "") +OBSERVABLE_WORDS = {"video_sha1": "video", "frame_hashes": "screens", "audio_sha1": "audio", + "state_sha1": "save state", "frame_sha1": "last frame"} + + +def setup_of(record: dict) -> dict: + """What told two runs of one test apart, each as the words a title + would use: the campaign check, the core build, the core options, the + clock, the number of files tried and what was compared.""" + build = str(record["core"].get("build_command") or "") + patched = re.search(r"\b(patch|edit(?:ed|ing)?)\b", build, re.IGNORECASE) + named = re.search(r"\b([A-Z]+[0-9]+)\b", build) if patched else None + check = record.get("check") or {} + first = (_words(record["subject"]) or [""])[0] + options = {} + for key, value in sorted(record["inputs"].get("options", {}).items()): + options[key] = f"{value} region" if key.endswith("_region") else f"{key.rsplit('_', 1)[-1]} {value}" + observable = str(check.get("observable") or "") + return { + "check": CHECK_NAMES.get(first, ""), + "build": (f"{named[1]} patch" if named else "patched build") if patched else "published build", + "options": options, + "clock": "fixed clock" if "FAKETIME" in record["inputs"].get("env", {}) else "running clock", + "files": f"{len(record['variants'])} files" if record["kind"] == "check" else "", + "observable": (observable.split(":", 1)[1] if observable.startswith("state:") + else OBSERVABLE_WORDS.get(observable, observable)), + } + + +def _features(item: dict) -> dict[str, str]: + """A test's setup as named features, the options one by one.""" + setup = item.get("setup") or {} + out = {name: str(setup.get(name) or "") for name in ("build", "clock", "files", "observable")} + for key, value in (setup.get("options") or {}).items(): + out[f"option:{key}"] = value + return out + + +def distinct_titles(items: list[dict]) -> dict[str, str]: + """A title per item: its game and campaign check, then what sets it + apart from the other tests of the same file, game and core. + + A test whose setup is the common one of its group is named by its game. + Another adds the features where it leaves the common setup (a patched + build, a running clock, another region). Two titles still alike take + every feature that varies in the group, then the subject's own words. + Nothing is cut. + """ + out: dict[str, str] = {} + peers: dict[tuple, list[dict]] = {} + for item in items: + game = short_content(item.get("content")) or short_title(item) + peers.setdefault((item.get("variable") or "", game, item["core"].get("file")), []).append(item) + for (_, game, _), group in peers.items(): + features = [_features(item) for item in group] + names = sorted({name for f in features for name in f}, + key=lambda n: (n.startswith("option:"), ["build", "clock", "files", "observable"].index(n) + if not n.startswith("option:") else 0, n)) + common = {} + for name in names: + values = [f.get(name, "") for f in features] + ranked = sorted(set(values), key=lambda v: -values.count(v)) + # A tie has no common value: every test names its own. + tied = len(ranked) > 1 and values.count(ranked[0]) == values.count(ranked[1]) + common[name] = None if tied else ranked[0] + varying = [n for n in names if len({f.get(n, "") for f in features}) > 1] + + def title(item: dict, own: dict[str, str], every: bool, varying: list[str] = varying, + common: dict[str, str] = common, game: str = game) -> str: + words = [(item.get("setup") or {}).get("check", "")] + for name in varying: + value = own.get(name, "") + if every or value != common[name]: + words.append(value or f"no {name.rsplit('_', 1)[-1]}") + return ", ".join([game, *[w for w in words if w]]) + + for item, own in zip(group, features): + out[item["id"]] = title(item, own, every=False) + for every in (True, None): + seen: dict[str, int] = {} + for item in group: + seen[out[item["id"]]] = seen.get(out[item["id"]], 0) + 1 + for item, own in zip(group, features): + if seen[out[item["id"]]] > 1: + if every: + out[item["id"]] = title(item, own, every=True) + else: + axis = short_title({**item, "variable": None, "content": None}) + out[item["id"]] = f"{out[item['id']]}, {axis}" + return out + + def short_host(host: dict) -> str: """The computer as a reader names it: the CPU model and the OS.""" cpu = str(host.get("cpu_model") or "unknown CPU") @@ -705,15 +794,15 @@ def _details(item: dict) -> list[str]: f"{item['repetitions']} repetition{'s' if item['repetitions'] != 1 else ''} each" if item["flags"]: reps += f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" - runs = "1 run" if item["runs"] == 1 else f"{item['runs']} runs, the latest shown" + runs = "" if item["runs"] == 1 else f", {item['runs']} runs, the latest shown" lines = [ _sentence_html("Claim", item.get("claim")), _sentence_html("Result", item.get("statement")), _sentence_html("Method", item.get("method")), _sentence_html("Proves", ", ".join(item.get("proves") or [])), - _sentence_html("Ran on", item["host"]), + _sentence_html("Ran on", item.get("host_short") or item["host"]), _sentence_html("Core build", build and f"{build}. {reps}", markup_ready=True), - f'

Full record, {runs}.

', + f'

Full record{runs}.

', ] return [line for line in lines if line] @@ -730,9 +819,9 @@ def core_label(item: dict) -> str: return Path(str(item["core"].get("file") or "")).name.split("_libretro")[0] -def _row(item: dict, page_host: str, main_core: str) -> list[str]: +def _row(item: dict, title: str, page_host: str, main_core: str) -> list[str]: elsewhere = (item.get("host_short") or item["host"]) != page_host - title = _cell(item.get("title") or short_title(item)) + title = _cell(title) if core_label(item) != main_core: title += f' {html.escape(core_label(item))}' return [ @@ -751,25 +840,41 @@ def _row(item: dict, page_host: str, main_core: str) -> list[str]: ] +def by_file(items: list[dict]) -> list[tuple[str, list[dict]]]: + """The items under the file each varies, in order of first appearance; + the runs that vary none last.""" + groups: dict[str, list[dict]] = {} + for item in items: + groups.setdefault(Path(str(item.get("variable") or "")).name, []).append(item) + return sorted(groups.items(), key=lambda kv: kv[0] == "") + + +def summary_line(items: list[dict], page_host: str) -> str: + """N tests, the confirmed and open counts when there are any, the last + run and its computer.""" + counts = tested_counts(items) + parts = [f"{counts['tests']} test{'s' if counts['tests'] != 1 else ''}"] + parts += [f"{counts[key]} {key}" for key in ("confirmed", "open") if counts[key]] + parts.append(f"last run {counts['last']} on {page_host}") + return " · ".join(parts) + "." + + def tested_section(name: str, items: list[dict]) -> list[str]: """The Tested block of an emulator page, empty when nothing was recorded. - The answer first: one summary line, then one row per test with its - result chip, a dot per variant and its date. What proves it (method, - claim, computer, core build, record) waits in each row's details. Tests - recorded before the boot oracle fold under one line, and benchmarks get - their own table at the end. + The answer first: one summary line, then the tests under the file each + one varies, a row per test with its result chip, a dot per variant and + its date. What proves it (method, claim, computer, core build, record) + waits in each row's details. Tests recorded before the boot oracle fold + under one line, and benchmarks get their own table at the end. """ if not items: return [] - counts = tested_counts(items) hosts = [i.get("host_short") or i["host"] for i in items] page_host = max(sorted(set(hosts)), key=hosts.count) cores = [core_label(i) for i in items] main_core = max(sorted(set(cores)), key=cores.count) - tests = f"{counts['tests']} test{'s' if counts['tests'] != 1 else ''}" - summary = (f"{tests} · {counts['confirmed']} confirmed · {counts['open']} open · " - f"last run {counts['last']} on {page_host}.") + titles = distinct_titles(items) links = (f'How · ' f'All tests · ' f'SQLite') @@ -778,18 +883,27 @@ def tested_section(name: str, items: list[dict]) -> list[str]: earlier = [i for i in ordered if i["kind"] != "bench" and i["status"] == "no verdict"] current = [i for i in ordered if i["kind"] != "bench" and i["status"] != "no verdict"] lines = ["## Tested", "", '
', - f'

{html.escape(summary)} {links}

'] + f'

{html.escape(summary_line(items, page_host))} {links}

'] if current or earlier: lines.append('") - for item in current: - lines += _row(item, page_host, main_core) + for file, group in by_file(current): + label = f"{html.escape(file)}" if file else "No file varied" + lines.append(f'

{label}

') + for item in group: + lines += _row(item, titles[item["id"]], page_host, main_core) + if current: + lines.append('

A dot per file tried, ' + 'the same colour for the same result. ' + " No file staged.

") if earlier: lines += ['
', f"{len(earlier)} earlier test{'s' if len(earlier) != 1 else ''}, " "before the boot oracle"] for item in earlier: - lines += _row(item, page_host, main_core) + file = Path(str(item.get("variable") or "")).name + title = f"{file}, {titles[item['id']]}" if file else titles[item["id"]] + lines += _row(item, title, page_host, main_core) lines.append("
") if benches: # One line: MDX reads a table split over lines as paragraphs. @@ -797,7 +911,7 @@ def tested_section(name: str, items: list[dict]) -> list[str]: "fpsstdevruns"] for item in benches: timing = item.get("timing") or {} - title = _cell(item.get("title") or short_title(item)) + title = _cell(titles[item["id"]]) if core_label(item) != main_core: title += f' {html.escape(core_label(item))}' table.append( diff --git a/tests/test_measurements.py b/tests/test_measurements.py index 5bf12a97b..890884fc7 100644 --- a/tests/test_measurements.py +++ b/tests/test_measurements.py @@ -221,7 +221,7 @@ class ReadAndIndex(Workspace): "claim", "method", "observable", "variable", "status", "statement", "content", "variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id", "runs", "record_url", "sqlite", "verdicts", "title", "host_short", "groups", - "proves", "timing"} + "proves", "timing", "setup"} for item in index["items"]: self.assertEqual(set(item), keys) alpha = {i["subject"]: i for i in index["items"] if i["emulator"] == "alpha"} @@ -430,19 +430,23 @@ class SitePages(unittest.TestCase): self.assertEqual(sm.tested_section("beetle_saturn", []), []) text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM])) self.assertTrue(text.startswith("## Tested\n")) - for expected in ("no difference", "Method: Twice per card. Every input fixed.", + for expected in ("same with every file", "Method: Twice per card. Every input fixed.", "Ran on: Test CPU, 12 CPUs", self.ITEM["record_url"], "1 flag raised", "2 runs", "downloads/measurements/beetle_saturn.sqlite", - "wiki/measurements/", "<image>", "1 test · 0 confirmed · 0 open"): + "wiki/measurements/", "<image>", "1 test · last run 2026-10-27", + '

mpeg_card.bin

', + 'class="rb-tested-legend"', "No file staged."): self.assertIn(expected, text) + self.assertNotIn("confirmed", text) + self.assertNotIn(" open", text) def item(self, **values: object) -> dict: return {**self.ITEM, "host_short": "Test CPU, Linux", **values} def test_summary_counts(self) -> None: items = [self.item(proves=["hle_fallback of a.bin"], open_findings=2), - self.item(subject="b", date="2026-10-28T00:00:00+00:00", open_findings=2), - self.item(kind="bench", status="measured", subject="c", + self.item(id="b", subject="b", date="2026-10-28T00:00:00+00:00", open_findings=2), + self.item(id="c", kind="bench", status="measured", subject="c", timing={"ms_per_frame": 1.5, "fps": 660.0, "stdev": 0.01})] self.assertEqual(sm.tested_counts(items), {"tests": 3, "confirmed": 1, "open": 2, "benchmarks": 1, "last": "2026-10-28"}) @@ -463,7 +467,8 @@ class SitePages(unittest.TestCase): "content": "Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00"}, "Lunar (issue 85 opening movie)"), ({"subject": "with-command-auth-device-and-command-get-9a211715", "variable": "mpeg_card.bin", - "content": "Redump: Lunar"}, "`mpeg_card.bin`, Lunar (with command auth device ...)"), + "content": "Redump: Lunar"}, + "`mpeg_card.bin`, Lunar (with command auth device and get)"), ] for item, title in cases: self.assertEqual(sm.short_title(item), title) @@ -471,6 +476,28 @@ class SitePages(unittest.TestCase): "os": "Linux-6.18.54.1-microsoft-standard-WSL2-x86_64"}), "Intel Core i9-13900K, Linux (WSL2)") + def test_rows_of_one_file_never_read_alike(self) -> None: + def item(rid: str, build: str, files: int, observable: str, clock: bool = True) -> dict: + record = {"core": {"build_command": build}, "subject": "mpeg-card-lunar", "kind": "check", + "check": {"observable": observable}, "variants": [{}] * files, + "inputs": {"options": {"beetle_saturn_mpeg_card": "enabled"}, + "env": {"FAKETIME": "x"} if clock else {}}} + return self.item(id=rid, content="Redump: Lunar - Silver Star Story", setup=sm.setup_of(record)) + items = [item("a", "make -j8", 11, "video_sha1"), + item("b", "make -j8 after editing cdb.c", 11, "video_sha1"), + item("c", "make -j10, with the CR2 patch to cdb.c", 3, "state:MPEG.AuthState"), + item("d", "make -j8", 11, "video_sha1", clock=False)] + titles = sm.distinct_titles(items) + self.assertEqual(titles, { + "a": "Lunar", + "b": "Lunar, patched build", + "c": "Lunar, CR2 patch, 3 files, MPEG.AuthState", + "d": "Lunar, running clock"}) + self.assertEqual(len(set(titles.values())), len(items)) + self.assertNotIn("...", " ".join(titles.values())) + # Alone in its file, a test is named by its game. + self.assertEqual(sm.distinct_titles(items[:1]), {"a": "Lunar"}) + def test_no_empty_sentence(self) -> None: text = "\n".join(sm.tested_section("beetle_saturn", [self.item(claim=None, method=None, statement="", kind="run", status="ran")])) @@ -488,16 +515,18 @@ class SitePages(unittest.TestCase): for _ in group["variants"]][:1] + [1, 0]) self.assertEqual([g["absent"] for g in groups], [False, False, True]) text = "\n".join(sm.tested_section("alpha", [self.item(groups=groups)])) - self.assertEqual(re.findall(r'rb-dot rb-dot-(\d)( rb-dot-empty)?"', text), + strip = text.split('class="rb-dots"', 1)[1].split("", 1)[0] + self.assertEqual(re.findall(r'rb-dot rb-dot-(\d)( rb-dot-empty)?"', strip), [("0", ""), ("1", ""), ("0", " rb-dot-empty")]) self.assertIn('aria-label="3 variants. v0: result A; v1: result B; v2: result A"', text) def test_the_details_hold_the_method_and_the_host(self) -> None: - text = "\n".join(sm.tested_section("beetle_saturn", [self.item()])) + text = "\n".join(sm.tested_section("beetle_saturn", [self.item(runs=1)])) details = text.split('
', 1)[1].split("
", 1)[0] self.assertIn("Twice per card", details) - self.assertIn("Test CPU, 12 CPUs", details) - self.assertIn(self.ITEM["record_url"], details) + self.assertIn("Ran on: Test CPU, Linux.", details) + self.assertIn(f'Full record.', details) + self.assertNotIn("1 run", details) def test_the_beetle_saturn_block_stays_short(self) -> None: stored = mr.read_records(REPO_ROOT) diff --git a/website/src/styles/retrobios.css b/website/src/styles/retrobios.css index 865b72e04..0d801de72 100644 --- a/website/src/styles/retrobios.css +++ b/website/src/styles/retrobios.css @@ -913,7 +913,7 @@ starlight-tabs [role='tab'] { white-space: nowrap; } .sl-markdown-content .rb-tested-row > summary:hover { background: var(--rb-surface); } .sl-markdown-content .rb-tested-test { font-weight: 600; overflow-wrap: anywhere; } .sl-markdown-content :is(.rb-tested-test, .rb-tested-date) small { color: var(--rb-text-secondary); font-weight: 400; } -.sl-markdown-content .rb-tested-date { color: var(--rb-text-secondary); font-variant-numeric: tabular-nums; } +.sl-markdown-content .rb-tested-date { color: var(--rb-text-secondary); font-variant-numeric: tabular-nums; white-space: nowrap; } .sl-markdown-content .rb-tested-more { margin: 0 0 0.75rem 1.75rem; padding: 0.5rem 0.75rem; @@ -938,12 +938,21 @@ starlight-tabs [role='tab'] { white-space: nowrap; } .sl-markdown-content .rb-dot-2 { --rb-dot: var(--rb-warn-fg); } .sl-markdown-content .rb-dot-3 { --rb-dot: var(--rb-err-fg); } .sl-markdown-content .rb-dot-4 { --rb-dot: var(--rb-muted-fg); } +.sl-markdown-content p.rb-tested-file { + margin: 0.9rem 0 0; + padding-bottom: 0.2rem; + border-bottom: 1px solid var(--rb-border-strong); + font-size: 0.85rem; + font-weight: 600; +} +.sl-markdown-content p.rb-tested-legend { margin-top: 0.5rem; color: var(--rb-text-secondary); font-size: 0.8rem; } +.sl-markdown-content p.rb-tested-legend .rb-dot { display: inline-block; vertical-align: -1px; } .sl-markdown-content table.rb-tested-bench { margin-top: 1rem; font-size: 0.875rem; } .sl-markdown-content table.rb-tested-bench td:not(:first-child) { font-variant-numeric: tabular-nums; text-align: right; } @media (max-width: 50rem) { .sl-markdown-content div.rb-tested-head { display: none; } .sl-markdown-content details.rb-tested-row > summary { - grid-template-columns: 1.25rem auto auto minmax(0, 1fr); + grid-template-columns: 1.25rem auto minmax(0, 1fr) auto; row-gap: 0.2rem; } .sl-markdown-content .rb-tested-row > summary::before { grid-row: 1; } From fabd431adb4f49f0661adb7afa09edeeb9afc4f8 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:02:03 +0100 Subject: [PATCH 4/6] feat: take the host probe lock per run core_probe takes probe.lock itself around one execution, load check included, and releases it when the run ends. A campaign pass never holds it between runs, so two passes interleave. RETROBIOS_PROBE_LOCK names another file, RETROBIOS_PROBE_LOCK_HELD says the caller holds it. Tests use their own lock file. --- scripts/campaign.py | 4 ++-- scripts/core_probe.py | 41 ++++++++++++++++++++++++++++++++++- tests/__init__.py | 5 +++++ tests/test_campaign.py | 46 ++++++++++++++++++++++++++++++++++++++++ tests/test_core_probe.py | 26 +++++++++++++++++++++++ wiki/tools.md | 10 +++++++++ 6 files changed, 129 insertions(+), 3 deletions(-) diff --git a/scripts/campaign.py b/scripts/campaign.py index ae0cd8eb4..c8cbd69da 100644 --- a/scripts/campaign.py +++ b/scripts/campaign.py @@ -303,9 +303,9 @@ def child_env(romfetch: bool = False) -> dict[str, str]: """What a program the campaign starts inherits: autorun's allowlist, never a token or a password. romfetch also gets its own settings (ROMFETCH_*), its account staying in its config file.""" - names: list[str] = [] + names: list[str] = [core_probe.PROBE_LOCK_ENV, core_probe.PROBE_LOCK_HELD_ENV] if romfetch: - names = [k for k in os.environ if k.startswith("ROMFETCH_")] + list(ROMFETCH_PASSED) + names += [k for k in os.environ if k.startswith("ROMFETCH_")] + list(ROMFETCH_PASSED) return autorun.safe_env(names) diff --git a/scripts/core_probe.py b/scripts/core_probe.py index bb35bf43e..1f979430e 100644 --- a/scripts/core_probe.py +++ b/scripts/core_probe.py @@ -21,6 +21,7 @@ import argparse import contextlib import ctypes import datetime as dt +import fcntl import hashlib import json import os @@ -35,7 +36,7 @@ import sys import tempfile import time import zlib -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Iterator, Sequence from dataclasses import dataclass, field from pathlib import Path @@ -73,6 +74,39 @@ PRINTF_SPEC = re.compile(r"%(?P[-+ #0]*)(?P\*|\d+)?(?:\.(?P\ r"(?Phh|h|ll|l|j|z|t|L)?(?P[diouxXeEfFgGaAcspn%])") +# One measurement at a time on the host: each core_probe execution takes this +# lock itself, its load check included, and releases it when it ends. A pass +# of many runs never holds it between them, so two passes interleave. +PROBE_LOCK_ENV = "RETROBIOS_PROBE_LOCK" +PROBE_LOCK_HELD_ENV = "RETROBIOS_PROBE_LOCK_HELD" +DEFAULT_PROBE_LOCK = Path.home() / ".local" / "state" / "retrobios" / "probe.lock" + + +def probe_lock_path() -> Path: + return Path(os.environ.get(PROBE_LOCK_ENV) or DEFAULT_PROBE_LOCK) + + +@contextlib.contextmanager +def host_probe_lock() -> Iterator[None]: + """The host's probe lock for one execution. A caller that already holds + it for this run says so with RETROBIOS_PROBE_LOCK_HELD.""" + if os.environ.get(PROBE_LOCK_HELD_ENV): + yield + return + path = probe_lock_path() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a") as handle: + try: + fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print(f"waiting for {path}: another measurement is running", file=sys.stderr) + fcntl.flock(handle, fcntl.LOCK_EX) + try: + yield + finally: + fcntl.flock(handle, fcntl.LOCK_UN) + + class ProbeError(RuntimeError): """A run that cannot produce a trustworthy result.""" @@ -1355,6 +1389,11 @@ def probe(args: argparse.Namespace) -> int: for problem in problems: print(f"ERROR: {problem}", file=sys.stderr) return 2 + with host_probe_lock(): + return _probe(args) + + +def _probe(args: argparse.Namespace) -> int: max_load = args.max_load if args.max_load is not None else (os.cpu_count() or 1) / 4 load_before = load_average() if args.record and load_before and load_before[0] > max_load: diff --git a/tests/__init__.py b/tests/__init__.py index 0190e539f..625337b3c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -16,6 +16,11 @@ import os from pathlib import Path os.environ.setdefault("RETROBIOS_OFFLINE", "1") +# A test never waits on, nor blocks, the measurements running on the host. +os.environ.setdefault( + "RETROBIOS_PROBE_LOCK", + str(Path(__file__).resolve().parent.parent / "tmp" / "tests" / "probe.lock"), +) (Path(__file__).resolve().parent.parent / "tmp" / "tests").mkdir( parents=True, exist_ok=True diff --git a/tests/test_campaign.py b/tests/test_campaign.py index ec5753227..071200d8b 100644 --- a/tests/test_campaign.py +++ b/tests/test_campaign.py @@ -1126,6 +1126,52 @@ class FakeCorePasses(Workspace): self.assertEqual({rep["verdict_evidence"]["rule"] for r in records for v in r["variants"] for rep in v["repetitions"]}, {"timeout"}) + def test_two_passes_interleave_their_runs(self) -> None: + """Each core_probe takes the host lock for its own run: a pass never + holds it between runs, so a second pass runs between them.""" + import threading + import time + + spans: dict[str, list[tuple[float, float]]] = {"a": [], "b": []} + other = Workspace(methodName="setUp") + other.setUp() + self.addCleanup(other.tearDown) + passes = [] + for name, workspace in (("a", self), ("b", other)): + c = FakeCorePasses.passing(workspace, "hle") if workspace is self else self.passing_in(workspace) + real = c.host.run + + def timed(argv: list[str], *args: object, _name: str = name, _real=real, **kwargs: object): + start = time.monotonic() + try: + return _real(argv, *args, **kwargs) + finally: + spans[_name].append((start, time.monotonic())) + c.host.run = timed + passes.append(c) + threads = [threading.Thread(target=c.run, args=(set(), None)) for c in passes] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + ends = {name: sorted(end for _, end in runs) for name, runs in spans.items()} + self.assertGreater(len(ends["a"]), 2) + self.assertGreater(len(ends["b"]), 2) + # Neither pass finished all its runs before the other started finishing its own. + self.assertLess(ends["b"][0], ends["a"][-1]) + self.assertLess(ends["a"][0], ends["b"][-1]) + + def passing_in(self, workspace: Workspace) -> campaign.Campaign: + workspace.put_game() + host = campaign.Host(loadavg=lambda: (0.0, 0.0, 0.0), free_gb=lambda path: 500.0, + cpu_count=1_000_000) + c = campaign.Campaign(workspace.paths, campaign.Settings( + check_repeat=2, probe_timeout=30, delays=(1,), echo=False), host, + lock_path=workspace.tmp / "auto.lock") + so = self.cores["hle"] + c.unit_cores = lambda *a, **k: [campaign.Core("fake", so, campaign.file_sha256(so))] + return c + def test_missing_content_leaves_the_checks_pending(self) -> None: c = self.passing(romfetch=str(self.tmp / "no-romfetch")) shutil.rmtree(self.game) diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index e927cbe2d..ead050b66 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -504,6 +504,32 @@ class FakeCoreTests(unittest.TestCase): sealed = measurement_records.seal({**record, "emulator": "fake", "subject": "s"}) self.assertEqual([e.message for e in Draft202012Validator(schema).iter_errors(sealed)], []) + def test_a_run_waits_for_the_host_lock_and_frees_it(self) -> None: + import fcntl + import time + + lock = self.tmp / "host.lock" + argv = [sys.executable, str(REPO_ROOT / "scripts" / "core_probe.py"), "run", "--core", + str(self.core), "--frames", "1", "--no-trace", "--work-dir", str(self.tmp / "work")] + env = {**os.environ, "RETROBIOS_PROBE_LOCK": str(lock)} + with lock.open("a") as held: + fcntl.flock(held, fcntl.LOCK_EX) + waiting = subprocess.Popen(argv, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True) + time.sleep(1.5) + self.assertIsNone(waiting.poll(), "the run went ahead while another held the lock") + # A caller that holds the lock for this run says so. + held_run = subprocess.run(argv, env={**env, "RETROBIOS_PROBE_LOCK_HELD": "1"}, + capture_output=True, text=True, timeout=60, check=False) + self.assertEqual(held_run.returncode, 0, held_run.stderr) + fcntl.flock(held, fcntl.LOCK_UN) + _, err = waiting.communicate(timeout=60) + self.assertEqual(waiting.returncode, 0, err) + self.assertIn("another measurement is running", err) + # Released at the end of the run: the lock is free again. + with lock.open("a") as again: + fcntl.flock(again, fcntl.LOCK_EX | fcntl.LOCK_NB) + def test_second_run_is_appended_beside_the_first(self) -> None: record_dir = checkout(self.tmp / "appended") argv = ["run", "--core", str(self.core), "--frames", "1", "--no-trace", "--max-load", "1000", diff --git a/wiki/tools.md b/wiki/tools.md index 622c37414..942ae6ca6 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -855,6 +855,15 @@ python scripts/core_probe.py check --core core.so --content disc.cue \ --observable frame_hashes --claim "..." --method "..." --repeat 2 --record ``` +One measurement runs at a time on the host. core_probe takes the host lock +itself, per run: `~/.local/state/retrobios/probe.lock` (`RETROBIOS_PROBE_LOCK` +names another file), from its load check to its record, released when the run +ends. A run that finds it held waits and says so. Never wrap a whole pass in +`flock`: a pass of many runs then holds the host for hours, while called +directly its runs interleave with everyone else's. A caller that holds the +lock around one core_probe call sets `RETROBIOS_PROBE_LOCK_HELD=1`. The tests +use their own lock file. + Each repetition runs in a fresh process under `strace -f -e trace=openat,open` when strace is installed. The worker starts with an empty environment and clears its own before it loads the core, keeping only the `--env` values. @@ -1046,6 +1055,7 @@ checks. The campaign's own `evidence:` writes do not count as a profile change. | Guard | Rule | |-------|------| | Lock | The host-wide autorun lock. Under the `campaign` job, autorun holds it and says so with `RETROBIOS_AUTO_LOCK_HELD` | +| Probe lock | Not held by the pass: each core_probe it starts takes the host probe lock for its own run, so two passes, or a pass and another agent's runs, interleave. Do not wrap `campaign.py run` in `flock` | | Environment | `core_probe.py`, git and `make` get autorun's allowlist, romfetch the same plus its `ROMFETCH_*` settings. None sees a token or a password | | Sandbox | Fails closed: without a working bubblewrap the pass is `refused` (exit 3) and nothing third-party runs, every probe is started with `--require-sandbox`, and a build is refused. `--allow-unsandboxed` overrides. Under autorun the refusal raises the `job:campaign` alert | | Load | Before each run, waits while the 1-minute load is above a quarter of the CPUs, up to `--max-load-wait` seconds, then the pass stops as `paused` | From 5fd691a7dbcb229e7f5b8484bdd8ac54512f0d93 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:02:15 +0100 Subject: [PATCH 5/6] docs: say which lock a campaign pass keeps --- wiki/tools.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/tools.md b/wiki/tools.md index 942ae6ca6..492ddf359 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1055,7 +1055,7 @@ checks. The campaign's own `evidence:` writes do not count as a profile change. | Guard | Rule | |-------|------| | Lock | The host-wide autorun lock. Under the `campaign` job, autorun holds it and says so with `RETROBIOS_AUTO_LOCK_HELD` | -| Probe lock | Not held by the pass: each core_probe it starts takes the host probe lock for its own run, so two passes, or a pass and another agent's runs, interleave. Do not wrap `campaign.py run` in `flock` | +| Probe lock | Not held by the pass: each core_probe it starts takes the host probe lock for its own run, so the pass and other measurements (pack_probe, a core_probe loop, a campaign of another checkout) interleave. Two passes of one checkout still exclude each other through the automation lock, since they share `state.json`. Do not wrap `campaign.py run` in `flock` | | Environment | `core_probe.py`, git and `make` get autorun's allowlist, romfetch the same plus its `ROMFETCH_*` settings. None sees a token or a password | | Sandbox | Fails closed: without a working bubblewrap the pass is `refused` (exit 3) and nothing third-party runs, every probe is started with `--require-sandbox`, and a build is refused. `--allow-unsandboxed` overrides. Under autorun the refusal raises the `job:campaign` alert | | Load | Before each run, waits while the 1-minute load is above a quarter of the CPUs, up to `--max-load-wait` seconds, then the pass stops as `paused` | From 9327d3f97f3dd4d76ea35774cc3d14d1bf1f0678 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:02:20 +0100 Subject: [PATCH 6/6] docs: two campaign passes share one state --- wiki/tools.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/tools.md b/wiki/tools.md index 492ddf359..08ae6443c 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1055,7 +1055,7 @@ checks. The campaign's own `evidence:` writes do not count as a profile change. | Guard | Rule | |-------|------| | Lock | The host-wide autorun lock. Under the `campaign` job, autorun holds it and says so with `RETROBIOS_AUTO_LOCK_HELD` | -| Probe lock | Not held by the pass: each core_probe it starts takes the host probe lock for its own run, so the pass and other measurements (pack_probe, a core_probe loop, a campaign of another checkout) interleave. Two passes of one checkout still exclude each other through the automation lock, since they share `state.json`. Do not wrap `campaign.py run` in `flock` | +| Probe lock | Not held by the pass: each core_probe it starts takes the host probe lock for its own run, so the pass and other measurements (pack_probe, a core_probe loop) interleave. Two campaign passes still exclude each other through the automation lock, since they share `.cache/campaign/state.json`. Do not wrap `campaign.py run` in `flock` | | Environment | `core_probe.py`, git and `make` get autorun's allowlist, romfetch the same plus its `ROMFETCH_*` settings. None sees a token or a password | | Sandbox | Fails closed: without a working bubblewrap the pass is `refused` (exit 3) and nothing third-party runs, every probe is started with `--require-sandbox`, and a build is refused. `--allow-unsandboxed` overrides. Under autorun the refusal raises the `job:campaign` alert | | Load | Before each run, waits while the 1-minute load is above a quarter of the CPUs, up to `--max-load-wait` seconds, then the pass stops as `paused` |