From c639a1b3e56afbe42b0b7e061293f9cd9a7d42df Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:01:10 +0100 Subject: [PATCH 1/3] feat: test the oracle against its own records Every reference block's run, judged again with the oracle as committed, boots in each repetition. Beetle Saturn's three BIOS lines are BIOS errors, the Japanese menu screens are error screens, and the PlayStation cores carry no error pattern. --- tests/test_core_probe.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index f9b3babb1..ec5491236 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -286,6 +286,45 @@ class OracleTests(unittest.TestCase): self.assertEqual(core_probe.oracle_for(data, "b" * 40, "Fake")["references"], []) self.assertEqual(core_probe.oracle_for(data, "a" * 40, "Nobody")["error_patterns"], ["shared"]) + def test_every_reference_judges_its_own_record_a_boot(self) -> None: + """Each reference block names the run it was taken from: judged + again with the oracle as it stands, every repetition of that run + boots, so a frame copied wrong or a stage mislabelled fails here.""" + data, _ = core_probe.load_oracle(core_probe.DEFAULT_ORACLE) + records = {s.record["id"]: s.record for s in measurement_records.read_records(REPO_ROOT)} + checked = 0 + for item in data.get("content") or (): + for library, block in item["references"].items(): + record = records.get(block["recorded_from"]) + self.assertIsNotNone(record, f"{item['name']} on {library}: {block['recorded_from']}") + self.assertEqual(record["inputs"]["content"]["sha1"], item["sha1"], item["name"]) + for rep in record["variants"][0]["repetitions"]: + oracle = core_probe.oracle_for(data, item["sha1"], rep["library_name"]) + verdict, evidence = core_probe.judge(rep, oracle) + self.assertEqual(verdict, "boots", f"{item['name']} on {library}: {evidence}") + checked += 1 + self.assertGreaterEqual(checked, 2 * 14) + + def test_the_repository_error_patterns_and_screens(self) -> None: + data, _ = core_probe.load_oracle(core_probe.DEFAULT_ORACLE) + saturn = core_probe.oracle_for(data, None, "Beetle Saturn") + for line in ('Cannot open BIOS file "/system/sega_101.bin".', + 'BIOS file "/system/sega_101.bin" is of an incorrect size.', + 'BIOS file "/system/sega_101.bin" could not be fully read (short or failed read).'): + rep = {"loaded": False, "logs": [{"frame": 0, "level": "error", "text": line}]} + self.assertEqual(core_probe.judge(rep, saturn)[0], "bios_error", line) + menu = {"Beetle PSX": "dfbfc299250c0983f86bbf951efd187441306d92", + "Beetle PSX HW": "dfbfc299250c0983f86bbf951efd187441306d92", + "SwanStation": "167fb7b6527e295b12dca5806a29856f5f31d401"} + for library, sha1 in menu.items(): + rep = {"loaded": True, "logs": [], "frames": 100, "warmup": 0, + "observables": {"frame_hashes": [[90, sha1]], "uniform_frames": []}} + verdict, evidence = core_probe.judge(rep, core_probe.oracle_for(data, None, library)) + self.assertEqual((verdict, evidence["rule"]), ("bios_error", "error_screen"), library) + # The PlayStation cores fall back to a built-in BIOS: nothing they log is an error. + for library in ("Beetle PSX", "SwanStation", "PCSX-ReARMed", "PCSX1"): + self.assertEqual(core_probe.oracle_for(data, None, library)["error_patterns"], []) + def test_the_repository_oracle_is_well_formed(self) -> None: import yaml data = yaml.safe_load(core_probe.DEFAULT_ORACLE.read_text(encoding="utf-8")) From 64af0a5d8336247c79c1ab47d3df611959854e52 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:08:30 +0100 Subject: [PATCH 2/3] feat: let a reference name its variant A reference taken from a check names the variant that ran the full set, and the test judges that variant's repetitions. --- tests/campaign/oracle.yml | 3 ++- tests/test_core_probe.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/campaign/oracle.yml b/tests/campaign/oracle.yml index 10e10f538..7724d4268 100644 --- a/tests/campaign/oracle.yml +++ b/tests/campaign/oracle.yml @@ -6,7 +6,8 @@ # retro_get_system_info gives): the reference frames a run with # the full BIOS set reaches. Each was recorded once # (`recorded_from`, the record id) and reviewed by looking at its -# PNG. `stage: bios` is a screen only the console's own BIOS +# PNG, from the variant `variant` names when that run was a +# check. `stage: bios` is a screen only the console's own BIOS # draws, `stage: game` a screen of the content. A run that reaches # one of them boots. # error_patterns log lines after which a core stops, read from its source diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index ec5491236..942d20b48 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -298,7 +298,11 @@ class OracleTests(unittest.TestCase): record = records.get(block["recorded_from"]) self.assertIsNotNone(record, f"{item['name']} on {library}: {block['recorded_from']}") self.assertEqual(record["inputs"]["content"]["sha1"], item["sha1"], item["name"]) - for rep in record["variants"][0]["repetitions"]: + # A check names the variant that ran the full set, a run has one. + label = block.get("variant") + chosen = next(v for v in record["variants"] if v["label"] == label) if label \ + else record["variants"][0] + for rep in chosen["repetitions"]: oracle = core_probe.oracle_for(data, item["sha1"], rep["library_name"]) verdict, evidence = core_probe.judge(rep, oracle) self.assertEqual(verdict, "boots", f"{item['name']} on {library}: {evidence}") From f2d474d86673d09275b49a5c052c6db0aad1e872 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:19:57 +0100 Subject: [PATCH 3/3] feat: add the PCSX2 missing BIOS dialog screen PCSX2 standalone shows a dialog and runs nothing without a BIOS: record 2026-10-27-fac0b52c9475 of the regions branch, reviewed on its PNG. --- tests/campaign/oracle.yml | 9 +++++++++ tests/test_core_probe.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/campaign/oracle.yml b/tests/campaign/oracle.yml index 7724d4268..cafaf7391 100644 --- a/tests/campaign/oracle.yml +++ b/tests/campaign/oracle.yml @@ -74,6 +74,15 @@ cores: - pattern: '^BIOS file ".*" could not be fully read' source_ref: mednafen/ss/ss.c:1978-1983 at 2ce77e20 error_screens: [] + "pcsx2": + note: >- + PCSX2 standalone (scripts/standalone_probe.py). Without a BIOS it shows a + dialog and runs nothing: the dialog is its error screen, captured as the + 1280x720 root window. + error_patterns: [] + error_screens: + - sha1: 46329c633ea7d0b63349261e4aab169e4f44d3f6 + shows: the dialog "PCSX2 requires a PlayStation 2 BIOS in order to run" "PCSX1": note: An HLE BIOS stands in when none is found, and the run logs nothing about it. error_patterns: [] diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index 942d20b48..889dea975 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -319,7 +319,8 @@ class OracleTests(unittest.TestCase): self.assertEqual(core_probe.judge(rep, saturn)[0], "bios_error", line) menu = {"Beetle PSX": "dfbfc299250c0983f86bbf951efd187441306d92", "Beetle PSX HW": "dfbfc299250c0983f86bbf951efd187441306d92", - "SwanStation": "167fb7b6527e295b12dca5806a29856f5f31d401"} + "SwanStation": "167fb7b6527e295b12dca5806a29856f5f31d401", + "pcsx2": "46329c633ea7d0b63349261e4aab169e4f44d3f6"} for library, sha1 in menu.items(): rep = {"loaded": True, "logs": [], "frames": 100, "warmup": 0, "observables": {"frame_hashes": [[90, sha1]], "uniform_frames": []}}