kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-01 00:27:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge proof lines and the BIOS used

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-10-28 08:08:39 +01:00
2 parents e95fe0ca9e 9c07dabfae
commit 702f7749c7
39 files changed
+1281 -1371
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/campaign/observations.yml
+20
View File
@@ -0,0 +1,20 @@
# What a maintainer saw in the runs an issue points at. The records keep
# frame and audio hashes, which say a run is the same as another, never what
# it showed: the sentence comes from watching the run. An entry names the
# records it was seen in (`records:`), or every record of a subject when the
# sentence is about all of them (`subjects:`). A run of an issue with no
# entry here and no boot verdict reaches no page.
# Read by scripts/sitemeasurements.py for the Proof lines of emulator pages.
issue85:
- said: the Lunar opening movie plays with sound
records: [2026-10-27-f17b2280e1af]
- said: Video CD play, pause, stop and play again work
records: [2026-10-27-a84574cf24a1, 2026-10-27-119c8e80edb5]
- said: next and previous track work on a three-track Video CD
records: [2026-10-27-51a78500620f]
- said: the 704x480 menu still of Super Gamer No. 5 shows
records: [2026-10-27-a9557cfd13d7]
# tests/test_measurements.py checks this one against the records: video
# and audio hashes equal across both core commits, card off and on.
- said: video and audio identical with and without the patch, card off and on, on 20 discs
subjects: [issue85-regression-card-off, issue85-regression-card-on]
tests/campaign/oracle.yml
+63 -1
View File
@@ -33,12 +33,42 @@ cores:
A missing BIOS logs "Firmware is missing" and loads the embedded
OpenBIOS, a wrong one logs "Firmware found but has invalid SHA1" and is
loaded anyway (libretro.c:849, 856, 3504-3511 at 7fea57a5).
fallback_patterns:
# No name of the region's list and no file of its sha1: the region's
# setting names the same first file, absent too, and OpenBIOS is mapped.
- pattern: '^Firmware is missing: '
source_ref: libretro.c:846-851, 3425-3511 and mednafen/settings.c:121-128 at 7fea57a5
load_patterns:
- pattern: '^Firmware found: (?P<path>.+)$'
source_ref: libretro.c:863-864 at 7fea57a5
# A file of the wrong sha1 is warned about and loaded all the same.
- pattern: '^Firmware found but has invalid SHA1: (?P<path>.+)$'
source_ref: libretro.c:853-861 at 7fea57a5
- pattern: '^Override firmware found: (?P<path>.+)$'
source_ref: libretro.c:758-761 at 7fea57a5
- pattern: '^Override firmware found but has invalid SHA1: (?P<path>.+)$'
source_ref: libretro.c:748-756 at 7fea57a5
error_patterns: []
error_screens:
- sha1: dfbfc299250c0983f86bbf951efd187441306d92
shows: the Japanese BIOS main menu, the disc refused (a US disc under a Japanese BIOS)
"Beetle PSX HW":
note: The same code as Beetle PSX, built with the hardware renderers.
fallback_patterns:
# No name of the region's list and no file of its sha1: the region's
# setting names the same first file, absent too, and OpenBIOS is mapped.
- pattern: '^Firmware is missing: '
source_ref: libretro.c:846-851, 3425-3511 and mednafen/settings.c:121-128 at 7fea57a5
load_patterns:
- pattern: '^Firmware found: (?P<path>.+)$'
source_ref: libretro.c:863-864 at 7fea57a5
# A file of the wrong sha1 is warned about and loaded all the same.
- pattern: '^Firmware found but has invalid SHA1: (?P<path>.+)$'
source_ref: libretro.c:853-861 at 7fea57a5
- pattern: '^Override firmware found: (?P<path>.+)$'
source_ref: libretro.c:758-761 at 7fea57a5
- pattern: '^Override firmware found but has invalid SHA1: (?P<path>.+)$'
source_ref: libretro.c:748-756 at 7fea57a5
error_patterns: []
error_screens:
- sha1: dfbfc299250c0983f86bbf951efd187441306d92
@@ -47,7 +77,18 @@ cores:
note: >-
"No BIOS image found for <region> region" is followed by the built-in
OpenBIOS (src/core/host_interface.cpp:182, src/core/system.cpp:678-682
at 631f0fb3).
at 631f0fb3). The file a region option names is loaded without a log
line, and an unknown image is dropped for OpenBIOS without one either:
those runs name no BIOS.
fallback_patterns:
- pattern: 'No BIOS image found for .* region'
source_ref: src/core/host_interface.cpp:180-184, src/core/system.cpp:678-682 at 631f0fb3
load_patterns:
- pattern: "Using BIOS '(?P<path>[^']+)'"
source_ref: src/core/host_interface.cpp:165-169 at 631f0fb3
# The other region's known image, when the console region has none.
- pattern: "Falling back to possibly-incompatible image '(?P<path>[^']+)'"
source_ref: src/core/host_interface.cpp:190-193 at 631f0fb3
error_patterns: []
error_screens:
- sha1: 167fb7b6527e295b12dca5806a29856f5f31d401
@@ -62,6 +103,15 @@ cores:
bios_patterns:
- pattern: '^found (US|EU|JP) BIOS file, crc32 '
source_ref: frontend/libretro.c:3646-3669 at c8816799
fallback_patterns:
- pattern: '^No BIOS files found\.'
source_ref: frontend/libretro.c:3896-3902 at c8816799
- pattern: '^Using HLE BIOS\.'
source_ref: frontend/libretro.c:3889-3894 at c8816799
load_patterns:
# Any region's file, the wrong one before HLE (psxmem.c:310).
- pattern: '^Loaded BIOS "(?P<path>[^"]+)"'
source_ref: libpcsxcore/psxmem.c:310-328 at c8816799
error_patterns: []
error_screens: []
"Beetle Saturn":
@@ -83,6 +133,18 @@ cores:
error_screens:
- sha1: 46329c633ea7d0b63349261e4aab169e4f44d3f6
shows: the dialog "PCSX2 requires a PlayStation 2 BIOS in order to run"
"Flycast":
note: >-
Dreamcast: without dc_boot.bin (or bios.bin) under dc/ it runs the reios
HLE BIOS. Read from the source, not yet seen in a run's log.
fallback_patterns:
- pattern: 'Did not load BIOS, using reios'
source_ref: core/emulator.cpp:614-619 at e88b0ced
load_patterns:
- pattern: 'Loaded (?P<path>.+) as bootrom'
source_ref: core/hw/flashrom/flashrom.cpp:56-64, core/hw/flashrom/nvmem.cpp:298-309 at e88b0ced
error_patterns: []
error_screens: []
"PCSX1":
note: An HLE BIOS stands in when none is found, and the run logs nothing about it.
error_patterns: []
tests/test_campaign.py
+23 -45
View File
@@ -34,7 +34,6 @@ import campaign
import campaign_profiles
import campaign_rules as rules
import measurement_records as mr
import sitemeasurements
TMP = REPO_ROOT / "tmp" / "tests"
@@ -192,6 +191,20 @@ class RuleTable(unittest.TestCase):
no_logo = run(rules.BOOTS, stages=("game",))
self.assertIsNone(rules.file_use(run(rules.BOOTS, stages=("game",)), NONE_BOOTS, no_logo))
def test_the_bios_the_log_names_settles_the_file_use(self) -> None:
present = run(rules.BOOTS, stages=("game",)) # no BIOS screen to compare with
fallback = rules.Run(rules.BOOTS, "r", "zeros", ("game",), "builtin")
loaded = rules.Run(rules.BOOTS, "r", "other", ("game",), "staged:bios.bin")
self.assertEqual(rules.file_use(fallback, NONE_BOOTS, present), rules.REFUSED)
self.assertEqual(rules.file_use(loaded, NONE_BOOTS, present), rules.USED)
self.assertIsNone(rules.file_use(run(rules.BOOTS, stages=("game",)), NONE_BOOTS, present))
record = {"id": "x", "variants": [{"label": "a", "verdict": "boots", "bios_used": "builtin",
"verdict_evidence": {"rule": "r", "detail": "d"}}]}
self.assertEqual(rules.run_of(record, "a").bios_used, "builtin")
builtin_none = rules.Run(rules.BOOTS, "n", "none", ("game",), "builtin")
(hle,) = rules.rule_need(builtin_none, PRESENT, {"name": "a"})
self.assertIn("on its built-in BIOS", hle.statement)
def test_validation_agrees_or_adds_size(self) -> None:
refused = run(rules.BOOTS, stages=("game",))
used = run(rules.HANG)
@@ -411,6 +424,15 @@ class StateAndResume(Workspace):
self.assertTrue(campaign.should_run({**done, "status": "pending"}, "a", "d"))
self.assertFalse(campaign.should_run({**done, "status": "failed"}, "a", "d"))
def test_a_record_judged_again_is_read_in_its_place(self) -> None:
lines = self.root / mr.records_path("fake")
lines.parent.mkdir(parents=True, exist_ok=True)
old = {"id": "2026-10-27-aaaaaaaaaaaa", "supersedes": None, "verdict": "old"}
new = {"id": "2026-10-28-bbbbbbbbbbbb", "supersedes": old["id"], "verdict": "new"}
lines.write_text("\n".join(json.dumps(r) for r in (old, new)) + "\n", encoding="utf-8")
self.assertEqual(self.make().read_record("fake", old["id"])["verdict"], "new")
self.assertIsNone(self.make().read_record("fake", "2026-10-27-cccccccccccc"))
def test_only_its_own_oracle_entries_make_a_check_run_again(self) -> None:
key = [hashlib.sha1(b"GAME").hexdigest(), "Fake"]
before = self.make().oracle_digest(key)
@@ -980,50 +1002,6 @@ class Registry(unittest.TestCase):
self.assertEqual(campaign.autorun.job_env(job)[campaign.LOCK_HELD_ENV], "1")
class SitePage(unittest.TestCase):
def test_last_pass_and_open_findings(self) -> None:
summary = {
"last_run": {"ended_at": "2026-10-27T10:00:00+00:00", "result": "complete", "detail": "",
"counts": {"done": 12, "failed": 1, "pending": 3},
"pending_reasons": {"standalone": 3},
"failed": [{"emulator": "fake", "check": "fake/x/y/bench", "reason": "timeout"}]},
"passes": 1, "applied": 2,
"open": [{"emulator": "fake", "file": "bios.bin", "field": "required", "status": "open",
"profile_says": True, "run_shows": False, "statement": "the core boots without it",
"evidence": ["2026-10-27-aaaaaaaaaaaa"], "links": ["https://x/l#L1"]}],
}
page = "\n".join(sitemeasurements.campaign_section(summary))
self.assertIn("12 tests done, 1 failed, 3 waiting", page)
self.assertIn("| standalone | 3 |", page)
self.assertIn("the profile says <code>true</code>, the run shows <code>false</code>", page)
self.assertIn("[2026-10-27-aaaaaaaaaaaa](https://x/l#L1)", page)
self.assertIn("<code>false</code>. The core boots without it. Records:", page)
self.assertEqual(sitemeasurements.campaign_section({"last_run": None}), [])
def test_observed_values_cannot_leave_their_place(self) -> None:
# A value comes from a core's output or a romfetch error: it may hold
# backticks, MDX braces and HTML.
hostile = "``}{import x}<script>alert(1)</script>`` | [a](javascript:x) *b*"
summary = {
"last_run": {"ended_at": "2026-10-27T10:00:00+00:00", "result": "paused", "detail": hostile,
"counts": {"done": 0, "failed": 1, "pending": 1},
"pending_reasons": {hostile: 1},
"failed": [{"emulator": hostile, "check": hostile, "reason": hostile}]},
"passes": 1, "applied": 0,
"open": [{"emulator": "fake", "file": hostile, "field": hostile, "status": "open",
"profile_says": {"k": hostile}, "run_shows": [hostile], "statement": hostile,
"evidence": ["2026-10-27-aaaaaaaaaaaa"], "links": [None]}],
}
page = "\n".join(sitemeasurements.campaign_section(summary))
for raw in ("<script", "{import", "}{", "``", "](javascript", "*b*", "| [a]"):
self.assertNotIn(raw, page)
self.assertIn("&lt;script&gt;alert(1)&lt;/script&gt;", page)
self.assertIn("&#123;import x&#125;", page)
# The JSON dumps are escaped too: no brace reaches the page as syntax.
self.assertNotIn("{", page)
self.assertNotIn("}", page)
@unittest.skipUnless(shutil.which("cc") and shutil.which("strace"), "no C compiler or no strace")
class FakeCorePasses(Workspace):
"""Whole passes against a core compiled from FAKE_CORE."""
tests/test_core_probe.py
+77 -2
View File
@@ -218,6 +218,27 @@ class OracleTests(unittest.TestCase):
self.assertEqual((verdict, evidence["stages"]), ("boots", ["bios", "game"]))
self.assertEqual(core_probe.judge(self.rep(frames((0, self.TITLE))), oracle)[1]["stages"], ["game"])
def test_the_bios_used_comes_from_the_log(self) -> None:
oracle = {**self.ORACLE, "fallback_patterns": [r"^Firmware is missing: "],
"load_patterns": [r"^Firmware found: (?P<path>.+)$"]}
cases = [
(["Searching for firmware checksum: 0555", "Firmware is missing: scph5501.bin"], "builtin"),
(["Firmware found: <system>/scph5501.bin", "Firmware SHA1: 0555"], "staged:scph5501.bin"),
# A file tried, then the fallback: the last line decides.
(["Firmware found: <system>/a.bin", "Firmware is missing: b.bin"], "builtin"),
(["nothing about the BIOS"], "unknown"),
]
for logs, expected in cases:
self.assertEqual(core_probe.bios_used(self.rep([], logs=logs), oracle)[0], expected, logs)
rep = self.rep(frames((10, self.TITLE)), logs=["Firmware is missing: scph5501.bin"])
core_probe.judge_repetition(rep, oracle)
self.assertEqual((rep["verdict"], rep["bios_used"]), ("boots", "builtin"))
self.assertTrue(rep["verdict_evidence"]["detail"].endswith("on the core's built-in BIOS"))
self.assertEqual(rep["verdict_evidence"]["bios_log"], "Firmware is missing: scph5501.bin")
variant = {"repetitions": [rep, {**rep, "bios_used": "staged:a.bin"}]}
core_probe.judge_variant(variant)
self.assertEqual(variant["bios_used"], "unknown")
def test_the_bios_screens_alone_are_not_a_boot(self) -> None:
# A console that shows its logo and then refuses the disc never reached the game.
verdict, evidence = core_probe.judge(
@@ -309,6 +330,29 @@ class OracleTests(unittest.TestCase):
checked += 1
self.assertGreaterEqual(checked, 2 * 14)
def test_the_repository_names_the_bios_each_core_ran(self) -> None:
data, _ = core_probe.load_oracle(core_probe.DEFAULT_ORACLE)
cases = {
"Beetle PSX": [("Firmware is missing: scph5501.bin", "builtin"),
("Firmware found: <system>/scph5501.bin", "staged:scph5501.bin"),
("Firmware found but has invalid SHA1: <system>/scph5501.bin", "staged:scph5501.bin")],
"Beetle PSX HW": [("Firmware is missing: scph5500.bin", "builtin")],
"SwanStation": [("[HostInterface] No BIOS image found for NTSC-U/C (US, Canada) region", "builtin"),
("[HostInterface] Using BIOS 'scph5501.bin': SCPH-5501", "staged:scph5501.bin"),
("[HostInterface] Falling back to possibly-incompatible image "
"'<system>/scph5500.bin': SCPH-5500", "staged:scph5500.bin")],
"PCSX-ReARMed": [('Loaded BIOS "<system>/scph5500.bin".', "staged:scph5500.bin"),
("No BIOS files found.", "builtin"), ("Using HLE BIOS.", "builtin")],
"Flycast": [("[BOOT]: Did not load BIOS, using reios", "builtin"),
("Loaded <system>/dc/dc_boot.bin as bootrom", "staged:dc/dc_boot.bin")],
"PCSX1": [("anything", "unknown")],
}
for library, lines in cases.items():
oracle = core_probe.oracle_for(data, None, library)
for text, expected in lines:
rep = {"logs": [{"frame": 0, "level": "info", "text": text}]}
self.assertEqual(core_probe.bios_used(rep, oracle)[0], expected, (library, text))
def test_the_repository_error_patterns_and_screens(self) -> None:
data, _ = core_probe.load_oracle(core_probe.DEFAULT_ORACLE)
saturn = core_probe.oracle_for(data, None, "Beetle Saturn")
@@ -335,7 +379,8 @@ class OracleTests(unittest.TestCase):
data = yaml.safe_load(core_probe.DEFAULT_ORACLE.read_text(encoding="utf-8"))
patterns = list(data.get("error_patterns") or ())
for core in (data.get("cores") or {}).values():
patterns += (core.get("error_patterns") or []) + (core.get("bios_patterns") or [])
patterns += [p for key in ("error_patterns", "bios_patterns", "fallback_patterns", "load_patterns")
for p in core.get(key) or []]
for screen in core.get("error_screens") or ():
self.assertRegex(screen["sha1"], r"^[0-9a-f]{40}$")
for pattern in patterns:
@@ -516,9 +561,39 @@ class FakeCoreTests(unittest.TestCase):
self.assertEqual(record["variants"][3]["system_files"],
{"card.bin": hashlib.sha1(b"\x00").hexdigest()})
self.assertEqual(record["inputs"]["oracle"]["error_patterns"], ["BIOS not found"])
self.assertEqual(record["record_version"], 4)
self.assertEqual(record["record_version"], 5)
self.assertEqual({v["bios_used"] for v in record["variants"]}, {"unknown"})
self.assert_schema(record)
def test_recorded_runs_are_judged_again_offline(self) -> None:
oracle = self.oracle_file()
record_dir = checkout(self.tmp / "rejudge")
code, first = self.probe(["run", "--core", str(self.core), "--frames", "4", "--frame-hash-every", "1",
"--content", str(self.tmp / "game.bin"), "--oracle", str(oracle),
"--no-trace", "--record", str(record_dir), "--max-load", "1000"])
self.assertEqual((code, first["variants"][0]["bios_used"]), (0, "unknown"))
# The core's fallback line becomes known: the same runs now name the BIOS.
oracle.write_text(oracle.read_text().replace(
" error_patterns: []\n", " error_patterns: []\n fallback_patterns:\n"
" - pattern: '^card absent'\n reason: test\n", 1), encoding="utf-8")
argv = ["rejudge", "--record", str(record_dir), "--oracle", str(oracle)]
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
self.assertEqual(core_probe.main(argv), 0)
lines = [json.loads(line) for line in (record_dir / "measurements" / "fake.jsonl").read_text().splitlines()]
self.assertEqual(len(lines), 2)
again = lines[1]
self.assertEqual((again["supersedes"], again["record_version"]), (first["id"], 5))
self.assertEqual(again["variants"][0]["bios_used"], "builtin")
self.assertEqual(again["variants"][0]["repetitions"][0]["logs"], first["variants"][0]["repetitions"][0]["logs"])
self.assertEqual(again["recorded_at"], first["recorded_at"])
self.assertIn("judged_at", again)
self.assertEqual(measurement_records.check_line(again), [])
self.assert_schema(again)
# Judged once: a second pass has nothing to add.
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
core_probe.main(argv)
self.assertEqual(len((record_dir / "measurements" / "fake.jsonl").read_text().splitlines()), 2)
def test_a_crash_and_a_hang_are_verdicts(self) -> None:
oracle = self.oracle_file()
code, record = self.probe(["run", "--core", str(self.core), "--frames", "4",
tests/test_measurements.py
+160 -152
View File
@@ -9,6 +9,7 @@ against HEAD.
from __future__ import annotations
import copy
import html
import json
import os
import re
@@ -216,12 +217,11 @@ class ReadAndIndex(Workspace):
index = self.built(self.tmp / "public").index
self.assertEqual({k: index[k] for k in ("schema_version", "kind", "repo", "commit", "records", "count")},
{"schema_version": 1, "kind": "measurements", "repo": REPO_URL,
"commit": COMMIT, "records": 3, "count": 3})
"commit": COMMIT, "records": 3, "count": 2})
keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind",
"claim", "method", "observable", "variable", "status", "statement", "content",
"variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id",
"runs", "record_url", "sqlite", "verdicts", "title", "host_short", "groups",
"proves", "timing", "setup"}
"runs", "record_url", "sqlite", "verdicts", "proves", "records"}
for item in index["items"]:
self.assertEqual(set(item), keys)
alpha = {i["subject"]: i for i in index["items"] if i["emulator"] == "alpha"}
@@ -229,9 +229,9 @@ class ReadAndIndex(Workspace):
self.assertEqual(alpha["card"]["system_names"], ["Sega Saturn"])
self.assertEqual(alpha["card"]["record_url"],
f"{REPO_URL}/blob/{COMMIT}/measurements/alpha.jsonl#L1")
self.assertEqual(alpha["speed"]["status"], "measured")
self.assertIn("ms per frame", alpha["speed"]["statement"])
self.assertIn("Test CPU", alpha["speed"]["host"])
# A benchmark stays in the records and the SQLite files, never in the index.
self.assertNotIn("speed", alpha)
self.assertNotIn("ms per frame", json.dumps(index))
written = json.loads((self.tmp / "public" / sm.INDEX_PATH).read_text())
self.assertEqual(written, index)
@@ -248,9 +248,30 @@ class ReadAndIndex(Workspace):
self.assertNotIn("differ", statement)
# A check on another observable keeps its comparison.
self.assertEqual(sm.result_of(full_record("alpha", "c", "2026-10-06T10:00:00+00:00"))[0], "differs")
groups = sm.variant_groups(record)
self.assertEqual([(g["variant"], g["result"], g["group"]) for g in groups],
[("v0", "works", 0), ("v1", "hang", 1)])
def test_a_boot_on_the_built_in_bios_is_named_so(self) -> None:
import core_probe # noqa: PLC0415
record = full_record("alpha", "region-x-game-others", "2026-10-28T10:00:00+00:00",
kind="run", observable="frame_hashes")
oracle = {"references": [{"sha1": "1" * 40, "stage": "game", "shows": "the title"}],
"error_patterns": [], "error_screens": [], "bios_patterns": [],
"fallback_patterns": [r"^Firmware is missing: "], "load_patterns": []}
for variant in record["variants"]:
for rep in variant["repetitions"]:
rep["logs"] = [{"frame": 0, "level": "info", "text": "Firmware is missing: scph5501.bin"}]
core_probe.judge_repetition(rep, oracle)
core_probe.judge_variant(variant)
self.assertEqual(record["variants"][0]["bios_used"], "builtin")
status, statement = sm.result_of(record)
self.assertEqual(status, sm.BOOTS_BUILTIN)
self.assertIn("built-in BIOS", statement)
# The proof line a profile field rests on names the BIOS the core ran.
record["variants"][0]["system_files"] = {}
line = sm.proof_line("required of scph5501.bin", [(sm.shown_of(record), ["../r.jsonl#L1"])])
self.assertEqual(html.unescape(line), "- required of scph5501.bin: with no BIOS, it boots on the "
"core's built-in BIOS ([record](../r.jsonl#L1)).")
self.assertNotIn("other region", line)
def test_links_name_the_last_commit_that_touched_the_records(self) -> None:
first = self.repository()
@@ -331,6 +352,24 @@ class ReadAndIndex(Workspace):
self.assertTrue(any("measurements/beta.jsonl:1" in e and "kind" in e for e in errors), errors)
self.assertIsNotNone(Draft202012Validator)
def test_a_version_5_record_names_the_bios_it_ran(self) -> None:
try:
from jsonschema import Draft202012Validator
except ImportError:
self.skipTest("jsonschema is not installed")
schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text())
record = mr.seal({**full_record("alpha", "card", "2026-10-28T10:00:00+00:00"), "record_version": 5})
errors = [e.message for e in Draft202012Validator(schema).iter_errors(record)]
self.assertIn("'bios_used' is a required property", errors)
for variant in record["variants"]:
variant["bios_used"] = "builtin"
for rep in variant["repetitions"]:
rep["bios_used"] = "staged:card.bin"
self.assertEqual([e.message for e in Draft202012Validator(schema).iter_errors(record)
if "bios_used" in e.message], [])
record["variants"][0]["bios_used"] = "the other region's"
self.assertTrue(list(Draft202012Validator(schema).iter_errors(record)))
def test_a_version_4_record_needs_its_verdicts(self) -> None:
try:
from jsonschema import Draft202012Validator # noqa: F401
@@ -423,160 +462,129 @@ class SitePages(unittest.TestCase):
"file": "x.so", "sha256": SHA256, "repo": None, "build": "make"},
"frontend_commit": None, "id": "2026-10-27-43527b763ace", "runs": 2,
"record_url": f"{REPO_URL}/blob/{'e' * 40}/measurements/beetle_saturn.jsonl#L4",
"sqlite": "downloads/measurements/beetle_saturn.sqlite",
"sqlite": "downloads/measurements/beetle_saturn.sqlite", "records": [],
}
def test_tested_block(self) -> None:
self.assertEqual(sm.tested_section("beetle_saturn", []), [])
text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM]))
self.assertTrue(text.startswith("## Tested\n"))
for expected in ("same with every file", "<strong>Method:</strong> Twice per card. Every input fixed.",
"<strong>Ran on:</strong> Test CPU, 12 CPUs", self.ITEM["record_url"],
"1 flag raised", "2 runs", "downloads/measurements/beetle_saturn.sqlite",
"wiki/measurements/", "&lt;image&gt;", "1 test · last run 2026-10-27",
'<p class="rb-tested-file"><code>mpeg_card.bin</code></p>',
'class="rb-tested-legend"', "No file staged."):
self.assertIn(expected, text)
self.assertNotIn("confirmed", text)
self.assertNotIn(" open", text)
def item(self, **values: object) -> dict:
return {**self.ITEM, "host_short": "Test CPU, Linux", **values}
return {**self.ITEM, **values}
def test_summary_counts(self) -> None:
items = [self.item(proves=["hle_fallback of a.bin"], open_findings=2),
self.item(id="b", subject="b", date="2026-10-28T00:00:00+00:00", open_findings=2),
self.item(id="c", kind="bench", status="measured", subject="c",
timing={"ms_per_frame": 1.5, "fps": 660.0, "stdev": 0.01})]
self.assertEqual(sm.tested_counts(items), {"tests": 3, "confirmed": 1, "open": 2, "benchmarks": 1,
"last": "2026-10-28"})
text = "\n".join(sm.tested_section("beetle_saturn", items))
self.assertIn("3 tests · 1 confirmed in the profile · 2 open findings · last run 2026-10-28 "
"on Test CPU, Linux.", text)
self.assertIn("<td>1.500</td><td>660.0</td>", text)
URL = f"{REPO_URL}/blob/{'e' * 40}/measurements/beetle_psx.jsonl"
def test_wrong_files_that_all_boot_fail_the_claim(self) -> None:
negative = self.item(subject="negative-scph5501-bin-x", status="boots")
self.assertIn(">accepts wrong files</span>", sm._chip(negative))
self.assertIn('rb-badge-info', sm._chip(negative))
self.assertIn(">works</span>", sm._chip(self.item(subject="alone-scph5501-bin-x", status="boots")))
def record(self, line: int, proves=(), shown=None, observed=None) -> dict:
return {"url": f"{self.URL}#L{line}", "proves": list(proves), "shown": shown, "observed": observed}
def test_the_file_label_links_its_card(self) -> None:
text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM],
{"mpeg_card.bin": "#file-mpeg-card-bin"}))
self.assertIn('<a href="#file-mpeg-card-bin"><code>mpeg_card.bin</code></a>', text)
def lines(self, *items: dict) -> list[str]:
return [html.unescape(line) for line in sm.proof_section(list(items)) if line.startswith("- ")]
def test_short_titles(self) -> None:
cases = [
({"subject": "mpeg-card-lunar-cr2-build", "variable": "mpeg_card.bin",
"content": "Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00"},
"mpeg_card.bin, Lunar (CR2 build)"),
({"subject": "negative-scph5501-bin-mednafen-psx-bust-a-move-2-arcade-edition-usa",
"variable": "scph5501.bin", "content": "Bust-A-Move 2 - Arcade Edition (USA)",
"core": {"file": "mednafen_psx_libretro.so"}},
"scph5501.bin, Bust-A-Move 2 (wrong files)"),
({"subject": "issue85-lunar-opening-movie", "variable": None,
"content": "Redump: Lunar - Silver Star Story Complete (Japan), T-27904G V1.00"},
"Lunar (issue 85 opening movie)"),
({"subject": "with-command-auth-device-and-command-get-9a211715", "variable": "mpeg_card.bin",
"content": "Redump: Lunar"},
"mpeg_card.bin, Lunar (with command auth device and get)"),
]
for item, title in cases:
self.assertEqual(sm.short_title(item), title)
self.assertEqual(sm.short_host({"cpu_model": "13th Gen Intel(R) Core(TM) i9-13900K",
"os": "Linux-6.18.54.1-microsoft-standard-WSL2-x86_64"}),
"Intel Core i9-13900K, Linux (WSL2)")
def test_one_line_per_claim(self) -> None:
"""Records that back the same claim collapse into one line: every
name strace saw opened in one list, each result once."""
alias = "aliases of scph5501.bin"
builtin = ["text", sm.proof_result([{"variant": "default", "verdict": sm.BOOTS_BUILTIN,
"bios_used": "builtin", "detail": "", "staged": []}])]
lines = self.lines(
self.item(subject="alias-b", records=[self.record(5, [alias], ["opened", "SCPH5501.BIN"])]),
self.item(subject="alias-a", records=[self.record(3, [alias], ["opened", "SCPH-5501.BIN"])]),
self.item(subject="none", records=[self.record(9, ["hle_fallback of scph5501.bin"], builtin),
self.record(7, ["hle_fallback of scph5501.bin"], builtin)]))
self.assertEqual(lines, [
f"- aliases of scph5501.bin: the core opened SCPH-5501.BIN, SCPH5501.BIN "
f"(records [1]({self.URL}#L3), [2]({self.URL}#L5)).",
f"- hle_fallback of scph5501.bin: with no BIOS, it boots on the core's built-in BIOS "
f"(records [1]({self.URL}#L7), [2]({self.URL}#L9)).",
])
many = self.lines(self.item(subject="s", records=[
self.record(n, ["required of a.bin"], ["text", "with no BIOS, it hangs"]) for n in range(1, 11)]))
self.assertEqual(many, [f"- required of a.bin: with no BIOS, it hangs ([10 records]({self.URL}))."])
# Two results for one claim stay on its one line, each with its records.
mixed = self.lines(self.item(subject="s", records=[
self.record(1, ["validation of a.bin"], ["text", "a.bin: boots. zeros: hangs"]),
self.record(2, ["validation of a.bin"], ["text", "with no BIOS, it boots"])]))
self.assertEqual(mixed, [f"- validation of a.bin: a.bin: boots. zeros: hangs ([record]({self.URL}#L1)). "
f"With no BIOS, it boots ([record]({self.URL}#L2))."])
def test_rows_of_one_file_never_read_alike(self) -> None:
def item(rid: str, build: str, files: int, observable: str, clock: bool = True) -> dict:
record = {"core": {"build_command": build}, "subject": "mpeg-card-lunar", "kind": "check",
"check": {"observable": observable}, "variants": [{}] * files,
"inputs": {"options": {"beetle_saturn_mpeg_card": "enabled"},
"env": {"FAKETIME": "x"} if clock else {}}}
return self.item(id=rid, content="Redump: Lunar - Silver Star Story", setup=sm.setup_of(record))
items = [item("a", "make -j8", 11, "video_sha1"),
item("b", "make -j8 after editing cdb.c", 11, "video_sha1"),
item("c", "make -j10, with the CR2 patch to cdb.c", 3, "state:MPEG.AuthState"),
item("d", "make -j8", 11, "video_sha1", clock=False)]
titles = sm.distinct_titles(items)
self.assertEqual(titles, {
"a": "Lunar",
"b": "Lunar, patched build",
"c": "Lunar, CR2 patch, 3 files, MPEG.AuthState",
"d": "Lunar, running clock"})
self.assertEqual(len(set(titles.values())), len(items))
self.assertNotIn("...", " ".join(titles.values()))
# Alone in its file, a test is named by its game.
self.assertEqual(sm.distinct_titles(items[:1]), {"a": "Lunar"})
def test_a_line_says_what_was_observed_or_is_left_out(self) -> None:
said = {"issue": "85", "said": "the Lunar opening movie plays with sound"}
lines = self.lines(
# A pre-oracle record with no opened file and no observation: nothing to state.
self.item(subject="alias-x", records=[self.record(1, ["aliases of a.bin"], None)]),
self.item(subject="issue85-vcd-player-boot", records=[self.record(2)]),
self.item(subject="issue85-lunar-opening-movie",
records=[self.record(3, observed=said), self.record(4)]))
self.assertEqual(lines, [f"- issue 85: the Lunar opening movie plays with sound ([record]({self.URL}#L3))."])
self.assertEqual(sm.proof_section([self.item(records=[self.record(1)])]), [])
def test_no_page_says_before_the_boot_oracle(self) -> None:
stored = mr.read_records(REPO_ROOT)
index = sm.build_index(stored, {"repo": REPO_URL, "commit": "e" * 40}, sm.load_profiles(REPO_ROOT / "emulators"),
None, sm.load_observations(REPO_ROOT / sm.OBSERVATIONS))
for emulator, items in sm.by_emulator(index).items():
text = "\n".join(sm.proof_section(items))
self.assertNotIn("boot oracle", text, emulator)
# One line per profile field: its records never spread over two lines.
fields = [line.split(":")[0] for line in text.splitlines()
if line.startswith("- ") and not line.startswith("- issue ")]
self.assertEqual(len(fields), len(set(fields)), emulator)
lines = [html.unescape(line) for line in sm.proof_section(sm.by_emulator(index).get("beetle_psx", []))
if line.startswith("- aliases of")]
self.assertEqual(len(lines), 1)
self.assertIn("the core opened ", lines[0])
def test_every_observation_names_records_that_exist(self) -> None:
observations = sm.load_observations(REPO_ROOT / sm.OBSERVATIONS)
stored = mr.read_records(REPO_ROOT)
known = {s.record["id"] for s in stored} | {s.record["subject"] for s in stored}
self.assertTrue(observations)
self.assertLessEqual(set(observations), known, set(observations) - known)
def test_the_issue_85_comparison_holds_in_the_records(self) -> None:
"""Video and audio equal on each disc across both core commits, card
off and on: the sentence the Proof line gives."""
said = sm.load_observations(REPO_ROOT / sm.OBSERVATIONS)["issue85-regression-card-on"]["said"]
self.assertIn("20 discs", said)
runs: dict[str, list[dict]] = {}
for entry in mr.read_records(REPO_ROOT):
if entry.record["subject"] in ("issue85-regression-card-off", "issue85-regression-card-on"):
runs.setdefault(entry.record["inputs"]["content"]["label"], []).append(entry.record)
if not runs:
self.skipTest("no issue 85 record")
self.assertEqual(len(runs), 20)
for disc, records in runs.items():
with self.subTest(disc=disc):
self.assertEqual(len({r["core"]["commit"] for r in records}), 2)
self.assertEqual({r["subject"] for r in records},
{"issue85-regression-card-off", "issue85-regression-card-on"})
seen = {(rep["observables"]["video_sha1"], rep["observables"]["audio_sha1"])
for r in records for v in r["variants"] for rep in v["repetitions"]}
self.assertEqual(len(seen), 1)
def test_the_removed_blocks_are_gone(self) -> None:
for name in ("tested_section", "region_section", "core_region_line", "campaign_section",
"coverage_section", "packs_section", "pack_tested_section", "bench_statement"):
self.assertFalse(hasattr(sm, name), name)
docs = REPO_ROOT / "website" / "src" / "content" / "docs"
pages = [*docs.glob("emulators/*.md*"), *docs.glob("systems/*.md*"),
*docs.glob("platforms/*.md*"), *docs.glob("measurements*.md*")]
if not pages:
self.skipTest("the site has not been generated")
for page in pages:
text = page.read_text(encoding="utf-8")
for removed in ("rb-tested", "rb-region-runs", "## Tested", "## Region check",
"ms per frame", "## Campaign", "## Coverage", "## Packs"):
self.assertNotIn(removed, text, f"{page.name}: {removed}")
def test_run_output_is_text_never_markup(self) -> None:
nasty = "`code` {expr} <script>alert(1)</script> [x](javascript:alert(1)) \"q\" 'q' **b**"
item = self.item(
claim=nasty, method=nasty, statement=nasty, host=nasty, host_short=nasty, content=nasty,
variable=f"{nasty}.bin", subject="s", title=nasty, proves=[nasty], runs=2, flags=1,
core={"library": nasty, "commit": "{`<x>`}", "file": f"{nasty}_libretro.so"},
groups=[{"variant": nasty, "group": 0, "result": nasty, "absent": False}],
record_url="javascript:alert(1)")
bench = self.item(id="b", kind="bench", status="measured", subject="bench", title=nasty,
content=nasty, record_url="java\nscript:x",
timing={"ms_per_frame": 1.0, "fps": 1.0, "stdev": 0.0})
earlier = self.item(id="e", status="no verdict", subject="e", content=nasty, claim=nasty)
text = "\n".join(sm.tested_section("alpha", [item, bench, earlier]))
self.assertNotIn("<script", text)
self.assertNotRegex(text, r'href="(?!https?://|/)')
self.assertNotIn("javascript:alert(1)\"", text)
self.assertNotRegex(text, r"[{}]")
self.assertNotIn("<strong>b</strong>", text)
self.assertNotIn("<code>code</code>", text)
self.assertNotRegex(text, r'="[^"]*"q"')
self.assertIn("&lt;script&gt;", text)
self.assertIn("&#123;expr&#125;", text)
def test_no_empty_sentence(self) -> None:
text = "\n".join(sm.tested_section("beetle_saturn", [self.item(claim=None, method=None, statement="",
kind="run", status="ran")]))
self.assertNotRegex(text, r"</strong>\s*\.|:\s*\.\s|`ran`\. \.")
self.assertNotIn("Claim:", text)
self.assertNotIn("Method:", text)
def test_dots_follow_the_result_groups(self) -> None:
record = full_record("alpha", "boot", "2026-10-06T10:00:00+00:00", values=("1" * 40, "2" * 40, "1" * 40))
record["check"]["variable"] = "card.bin"
record["variants"][2]["system_files"] = {"card.bin": None}
groups = sm.variant_groups(record)
self.assertEqual([g["group"] for g in groups],
[i for i, group in enumerate(record["check"]["result"]["groups"])
for _ in group["variants"]][:1] + [1, 0])
self.assertEqual([g["absent"] for g in groups], [False, False, True])
text = "\n".join(sm.tested_section("alpha", [self.item(groups=groups)]))
strip = text.split('class="rb-dots"', 1)[1].split("</span></span>", 1)[0]
self.assertEqual(re.findall(r'rb-dot rb-dot-(\d)( rb-dot-empty)?"', strip),
[("0", ""), ("1", ""), ("0", " rb-dot-empty")])
self.assertIn('aria-label="3 variants. v0: result A. v1: result B. v2: result A"', text)
def test_the_details_hold_the_method_and_the_host(self) -> None:
text = "\n".join(sm.tested_section("beetle_saturn", [self.item(runs=1)]))
details = text.split('<div class="rb-tested-more">', 1)[1].split("</div>", 1)[0]
self.assertIn("Twice per card", details)
self.assertIn("<strong>Ran on:</strong> Test CPU, Linux.", details)
self.assertIn(f'<a href="{self.ITEM["record_url"]}">Full record</a>.', details)
self.assertNotIn("1 run", details)
def test_the_beetle_saturn_block_stays_short(self) -> None:
stored = mr.read_records(REPO_ROOT)
profiles = sm.load_profiles(REPO_ROOT / "emulators")
index = sm.build_index(stored, {"repo": REPO_URL, "commit": COMMIT}, profiles)
items = sm.by_emulator(index).get("beetle_saturn")
if not items:
self.skipTest("no beetle_saturn record")
block = "\n".join(sm.tested_section("beetle_saturn", items))
# What a reader sees before opening anything: the closed details hide their body.
shown = re.sub(r'<div class="rb-tested-more">.*?</div>', " ", block, flags=re.S)
shown = re.sub(r'<details class="rb-tested-earlier">.*?</details>\s*</details>', " ", shown,
flags=re.S)
shown = re.sub(r"<[^>]+>", " ", shown)
self.assertLess(len(re.findall(r"[A-Za-z0-9][\w.'-]*", shown)), 120, shown)
verdicts = [{"variant": nasty, "verdict": nasty, "bios_used": f"staged:{nasty}", "detail": nasty},
{"variant": nasty, "verdict": "boots", "bios_used": "builtin", "detail": nasty}]
line = sm.proof_line(nasty, [(["text", sm.proof_result(verdicts)], ["javascript:alert(1)"]),
(["opened", nasty], ["javascript:alert(1)"])])
self.assertNotIn("<script", line)
self.assertNotIn("(javascript:", line)
self.assertNotRegex(line, r"[{}`*]")
self.assertNotRegex(line, r"\[")
self.assertNotIn("](", line)
self.assertIn("&lt;script&gt;", line)
def test_hosts_table_on_the_method_page(self) -> None:
page = sm.with_hosts(f"x\n\n{sm.HOSTS_MARKER}\n",
tests/test_pack_probe.py
+7 -52
View File
@@ -218,43 +218,14 @@ class Line(unittest.TestCase):
class SitePages(unittest.TestCase):
"""The Measurements page and the platform pages show the latest probe."""
"""A pack probe runs before a release: its records stay in measurements/
and reach no page."""
def test_latest_line_per_probe_with_its_link(self) -> None:
def test_the_pack_blocks_are_gone(self) -> None:
import sitemeasurements as sm
root = TMP / "site"
shutil.rmtree(root, ignore_errors=True)
root.mkdir(parents=True)
first = Line.line(None)
later = {**first, "probed_at": "2026-10-28T12:00:00+00:00", "verdict": "boots"}
other = {**first, "platform": "batocera", "install": "installer"}
for line in (first, later, other):
pp.append_line(line, root)
items = sm.pack_summary(root, {"repo": "https://example.org/r", "commit": "c" * 40})
self.assertEqual([(i["platform"], i["install"], i["verdict"]) for i in items],
[("batocera", "installer", "unknown"), ("retroarch", "pack", "boots")])
self.assertEqual(items[1]["record_url"],
f"https://example.org/r/blob/{'c' * 40}/measurements/pack-probes.jsonl#L2")
self.assertEqual(items[1]["core"], "mednafen_saturn")
explorer = "\n".join(sm.packs_section(items))
self.assertIn("| Platform | System |", explorer)
self.assertIn("| retroarch | sega-saturn | pack | <code>mednafen&#95;saturn</code> | Game (Japan) "
"| `boots` | 1 | [line](", explorer)
page = sm.pack_tested_section(sm.by_platform({"packs": items})["retroarch"])
text = "\n".join(page)
self.assertEqual(page[0], "## Tested")
self.assertIn("1 game start with this BIOS folder as you install it · 1 work · last run 2026-10-28.",
text)
self.assertIn("wiki/measurements/#packs", text)
# The emulator pages' row markup and chip, the install path in the third column.
self.assertIn('<span class="rb-tested-result"><span class="rb-badge rb-badge-success">works</span>',
text)
self.assertIn('<span class="rb-tested-variants">pack</span>', text)
self.assertIn("sega-saturn, Game <small>mednafen&#95;saturn</small>", text)
self.assertIn("<strong>Files the core opened:</strong> sega&#95;101.bin (sega&#95;101.bin).", text)
self.assertIn("<strong>Findings:</strong> missing&#95;declared x.bin.", text)
self.assertNotIn("| retroarch |", text)
for name in ("pack_summary", "packs_section", "pack_tested_section", "by_platform"):
self.assertFalse(hasattr(sm, name), name)
def test_pack_runs_stay_off_the_emulator_page(self) -> None:
import sitemeasurements as sm
@@ -266,30 +237,14 @@ class SitePages(unittest.TestCase):
self.assertEqual([i["subject"] for i in sm.by_emulator(index)["beetle_saturn"]],
["boot-sega-101-bin"])
def test_run_output_is_inert(self) -> None:
"""A file name or log line from a third-party core reaches the page
as text: no tag, link, code span or MDX expression comes out of it."""
def test_a_markdown_link_keeps_only_safe_targets(self) -> None:
"""A Markdown link in site text keeps only http(s) and relative targets."""
import sitemarkup
import sitemeasurements as sm
hostile = "`x` {alert(1)} <script>alert(1)</script> [x](javascript:alert(1)) \"q\" 'q' **b**"
item = {"platform": "retroarch", "platform_version": "1", "install": "pack",
"system": hostile, "emulator": hostile, "core": hostile, "content": hostile,
"verdict": "boots", "date": "2026-10-28T00:00:00+00:00", "opened": 1, "findings": 1,
"id": "2026-10-28-0123456789ab", "record_url": "https://example.org/r#L1",
"files": [hostile], "finding_list": [hostile], "detail": hostile}
text = "\n".join(sm.pack_tested_section([item]))
for raw in ("<script", "{alert", "`x`", "[x](", "<code>", "<strong>b", "**b",
"href=\"javascript", '"q"', "'q'"):
self.assertNotIn(raw, text)
self.assertIn("&lt;script&gt;", text)
self.assertIn("&#123;alert(1)&#125;", text)
# A Markdown link in other site text keeps only http(s) and relative targets.
self.assertEqual(sitemarkup.inline_html("[x](javascript:alert(1))"), "[x](javascript:alert(1))")
self.assertEqual(sitemarkup.inline_html('[x](https://a.org/"onmouseover=1)'),
'<a href="https://a.org/&quot;onmouseover=1">x</a>')
self.assertEqual(sitemarkup.inline_html("[x](../a.md)"), '<a href="../a.md">x</a>')
self.assertEqual(sm.pack_tested_section([]), [])
class Lock(unittest.TestCase):
tests/test_region_check.py
+46 -68
View File
@@ -27,9 +27,10 @@ PROFILE = {
def record(rid: str, staged: list[str], opened: list[str], hashes: list, verdict: str = "unknown",
loaded: bool = True) -> dict:
loaded: bool = True, bios_used: str = "unknown") -> dict:
return {"id": rid, "variants": [{
"label": "default", "verdict": verdict, "system_files": {name: "0" * 40 for name in staged},
"label": "default", "verdict": verdict, "bios_used": bios_used,
"system_files": {name: "0" * 40 for name in staged},
"repetitions": [{"loaded": loaded, "files_opened": [{"path": p} for p in opened],
"observables": {"frame_hashes": hashes}}]}]}
@@ -145,8 +146,30 @@ class Outcome(unittest.TestCase):
"own": record("o", ["scph5502.bin"], [], [[0, "a"]], "boots"),
"others": record("x", ["scph5501.bin"], ["scph5501.bin"], [[0, "a"]], "boots")}
result = rc.outcome(records, self.ENTRIES, "europe")
# The log names no BIOS: the boot is not said to be the other region's.
self.assertEqual(rc.sentence("europe", result),
"The European BIOS boots the European game, and so do the other regions' BIOS.")
"The European BIOS boots the European game, and with the others' files on a "
"BIOS its log does not name.")
records["others"] = record("x", ["scph5501.bin"], ["scph5501.bin"], [[0, "a"]], "boots",
bios_used="staged:scph5501.bin")
self.assertEqual(rc.sentence("europe", rc.outcome(records, self.ENTRIES, "europe")),
"The European BIOS boots the European game, and so does scph5501.bin, "
"another region's BIOS.")
def test_a_boot_on_the_built_in_bios_is_never_another_regions(self) -> None:
"""Beetle PSX logs "Firmware is missing" and maps OpenBIOS: the other
regions' files were opened, hashed and refused."""
records = {"full": record("f", self.ALL, self.ALL, [[0, "a"]], "boots",
bios_used="staged:scph5501.bin"),
"own": record("o", ["scph5501.bin"], ["scph5501.bin"], [[0, "a"]], "boots",
bios_used="staged:scph5501.bin"),
"others": record("x", ["scph5500.bin", "scph5502.bin"], ["scph5500.bin", "scph5502.bin"],
[[0, "a"]], "boots", bios_used="builtin")}
result = rc.outcome(records, self.ENTRIES, "north-america")
text = rc.sentence("north-america", result)
self.assertEqual(text, "The US BIOS boots the US game, and on its built-in BIOS with the "
"others' files refused.")
self.assertNotIn("so do the other regions", text)
def test_no_own_run(self) -> None:
self.assertEqual(rc.sentence("japan", rc.outcome({}, self.ENTRIES, "japan")),
@@ -205,83 +228,38 @@ class RequiredFromTheRegionRuns(unittest.TestCase):
self.assertTrue(ids)
for rid in ids:
record = records[rid]
self.assertEqual(record["record_version"], 4)
self.assertEqual(record["record_version"], 5)
variant = record["variants"][0]
self.assertNotIn(entry["name"], variant["system_files"])
self.assertEqual(variant["verdict"], "boots")
# The boot was on OpenBIOS, never on another region's file.
self.assertEqual(variant["bios_used"], "builtin")
self.assertIn("OpenBIOS", entry["note"])
self.assertIn("game", variant["verdict_evidence"].get("stages", []))
class SitePages(unittest.TestCase):
def test_region_section_links_every_record(self) -> None:
import sitemeasurements as sm
records = {"full": record("f", Outcome.ALL, ["scph5500.bin"], [[0, "a"]], "boots"),
"own": record("o", ["scph5500.bin"], ["scph5500.bin"], [[0, "a"]], "boots"),
"others": record("x", Outcome.ALL[1:], [], [[0, "b"]], "bios_error")}
result = rc.outcome(records, Outcome.ENTRIES, "japan")
result.update(core="mednafen_psx", game="Puzzle Bobble 2 (Japan)",
sentence=rc.sentence("japan", result))
for label, item in result["variants"].items():
item["url"] = f"https://example.org/{label}"
text = "\n".join(sm.region_section([result]))
self.assertIn("- **mednafen&#95;psx**: Each region's game boots on its own BIOS only.", text)
self.assertIn("The Japanese BIOS boots the Japanese game, the other regions&#x27; BIOS do not.", text)
# The per-game lines and their records wait in a closed fold.
self.assertIn('<details class="rb-region-runs">', text)
self.assertIn('<a href="https://example.org/own">its own region&#x27;s BIOS only</a>', text)
self.assertIn("<code>scph5500.bin</code>", text)
self.assertNotIn("reference frames are recorded", text)
self.assertEqual(sm.region_section([]), [])
def test_the_emulator_page_folds_region_runs_into_one_line(self) -> None:
def test_a_region_run_reaches_a_page_only_as_the_proof_of_a_field(self) -> None:
"""No page shows region outcomes. The Beetle PSX page names the runs
its required: false rests on, each with the BIOS the core ran."""
import measurement_records as mr
import sitemeasurements as sm
stored = mr.read_records(ROOT)
index = sm.build_index(stored, {"repo": sm.REPO, "commit": "0" * 40}, sm.load_profiles(ROOT / "emulators"))
index = sm.build_index(stored, {"repo": sm.REPO, "commit": "0" * 40}, sm.load_profiles(ROOT / "emulators"),
None, sm.load_observations(ROOT / sm.OBSERVATIONS))
items = sm.by_emulator(index).get("beetle_psx") or []
regional = [i for i in items if sm.REGION_SUBJECT.match(i["subject"])]
if not regional:
self.skipTest("no region record")
text = "\n".join(sm.tested_section("beetle_psx", items))
self.assertIn(f"Region check: {len(regional)} runs of 3 games, read on the system page.", text)
for item in regional:
self.assertNotIn(item["record_url"], text)
def test_coverage_names_tested_and_uncovered_systems(self) -> None:
import sitemeasurements as sm
cov = {"systems": [{"system": "sony-playstation", "tested": True, "games": [
{"name": "A (USA)", "region": "north-america", "emulators": ["beetle_psx"]}]},
{"system": "sega-32x", "tested": False, "games": [
{"name": "B (World)", "region": None, "emulators": []}]}],
"uncovered": [{"system": "msx", "want": "Gradius", "reason": "No MSX catalog.",
"romfetch": {"exit": 1, "code": "unknown_system"}}]}
text = "\n".join(sm.coverage_section(cov, {"sony-playstation": "PlayStation"}))
self.assertIn("1 of 2 listed systems have a recorded run, and 1 system has no test game", text)
self.assertIn("| PlayStation | A (USA) | beetle&#95;psx |", text)
self.assertIn("| sega-32x | B (World) | not yet: no run recorded |", text)
self.assertIn("exit 1, <code>unknown&#95;system</code>", text)
def test_one_catalog_name_on_two_systems_counts_only_where_it_ran(self) -> None:
import measurement_records as mr
import sitemeasurements as sm
label = "Bust-A-Move 2 - Arcade Edition (USA)"
stored = [mr.Stored("beetle_psx", 1, {"inputs": {"content": {"label": label}}})]
cov = sm.coverage(ROOT, stored, {"beetle_psx": {"systems": ["sony-playstation"]}})
by_system = {item["system"]: item for item in cov["systems"]}
self.assertTrue(by_system["sony-playstation"]["tested"])
self.assertFalse(by_system["sega-saturn"]["tested"])
def test_coverage_reads_the_real_content_list(self) -> None:
import sitemeasurements as sm
cov = sm.coverage(ROOT, [])
systems = {item["system"] for item in cov["systems"]}
self.assertIn("sony-playstation", systems)
self.assertTrue(all(item["romfetch"].get("exit") for item in cov["uncovered"]))
if not items:
self.skipTest("no beetle_psx record")
text = "\n".join(sm.proof_section(items))
self.assertNotIn("Region check", text)
lines = [line for line in text.splitlines() if line.startswith("- ")]
required = [line for line in lines if line.startswith("- required of scph55")]
self.assertEqual(len(required), 3)
for line in required:
# The run staged only the other regions' files and booted OpenBIOS.
self.assertIn("it boots on the core&#x27;s built-in BIOS", line)
self.assertIn(", it boots", line)
if __name__ == "__main__":
tests/test_site_r3_emulators.py
-9
View File
@@ -192,15 +192,6 @@ class AnElementBudget(unittest.TestCase):
self.assertLess(page.elements, validate_site.MAX_PAGE_ELEMENTS)
class TestedFilesLeadToTheirCard(unittest.TestCase):
def test_a_name_or_alias_finds_the_card(self):
files = [{"name": "scph5501.bin", "aliases": ["SCPH-5501.BIN"]}, {"name": "other.bin"}]
hrefs = gs._tested_file_hrefs("x", files)
self.assertEqual(hrefs["scph5501.bin"], "#file-scph5501-bin")
self.assertEqual(hrefs["scph-5501.bin"], "#file-scph5501-bin")
self.assertEqual(hrefs["other.bin"], "#file-other-bin")
if __name__ == "__main__":
unittest.main()
tests/test_workflow_paths.py
+1 -1
View File
@@ -155,7 +155,7 @@ class ThePullRequestBuildsTheSite(unittest.TestCase):
}
self.assertIn("wiki/**", inputs)
# A path the deploy names inside a folder validate.yml watches
# whole (tests/campaign/content.yml under tests/**) is watched.
# whole (tests/campaign/observations.yml under tests/**) is watched.
unwatched = [path for path in inputs - watched
if not any(fnmatch.fnmatch(path, pattern) for pattern in watched)]
self.assertEqual(sorted(unwatched), [])