chore: merge main into the findings review

This commit is contained in:
Abdessamad Derraz committed 2026-10-28 09:28:29 +01:00
commit 448989665a
39 files changed
+1303 -1371

No files matched your search

+10 -4
View File
@@ -1288,15 +1288,21 @@ class Campaign:
self.settle(unit.emulator, content, core, index, entry, [verdict])
def read_record(self, emulator: str, rid: str | None) -> dict | None:
"""A record by its id, or the record that supersedes it: a run
judged again offline is read as judged last."""
if not rid:
return None
path = self.paths.root / mr.records_path(emulator)
if not path.is_file():
return None
for line in path.read_text(encoding="utf-8").splitlines():
if f'"id":"{rid}"' in line:
return json.loads(line)
return None
records = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()]
by_id = {r["id"]: r for r in records}
successor = {r["supersedes"]: r["id"] for r in records if r.get("supersedes")}
seen = set()
while rid in successor and rid not in seen:
seen.add(rid)
rid = successor[rid]
return by_id.get(rid)
def judged(self, emulator: str, rid: str) -> bool:
"""True for a record the boot oracle judged: only those prove a field."""
+13 -4
View File
@@ -25,8 +25,9 @@ name (zeros of its size, its first half, another region's dump).
copies of one file that boot and fail - queued
"Fails" is bios_error, crash or hang. A wrong file is refused when the
core reports a BIOS error, or when it boots without the BIOS screens the
right file shows (it fell back to a built-in BIOS). It is used when it
core reports a BIOS error, when its log says it ran its built-in BIOS
(bios_used builtin), or when it boots without the BIOS screens the right
file shows. It is used when the log names the file it loaded. It is used when it
turns the run into a hang or a crash that the run without any file does not
show. Anything else settles nothing. A check the entry declares with
`on_mismatch: warn` is one the code only logs: a wrong file that runs
@@ -66,6 +67,7 @@ class Run:
record_id: str
label: str
stages: tuple[str, ...] = ()
bios_used: str = "unknown"
@property
def shows_bios(self) -> bool:
@@ -103,7 +105,8 @@ def run_of(record: dict, label: str) -> Run:
oracle has none, and reads as unknown."""
item = variant(record, label)
evidence = item.get("verdict_evidence") or {}
return Run(item.get("verdict", UNKNOWN), record["id"], label, tuple(evidence.get("stages") or ()))
return Run(item.get("verdict", UNKNOWN), record["id"], label, tuple(evidence.get("stages") or ()),
item.get("bios_used", "unknown"))
def ids(*runs: Run) -> tuple[str, ...]:
@@ -140,8 +143,9 @@ def rule_need(none: Run, present: Run, entry: dict, used: bool = True) -> list[V
if none.verdict == BOOTS and not used:
return []
if none.verdict == BOOTS:
on = " on its built-in BIOS" if none.bios_used == "builtin" else ""
out = [Verdict("boots_without_file", "hle_fallback", True,
f"the core boots the content with no BIOS at all, {name} included",
f"the core boots the content{on} with no BIOS at all, {name} included",
ids(none), True)]
if entry.get("required") is True:
out.append(Verdict(
@@ -169,6 +173,11 @@ def file_use(wrong: Run, none: Run, present: Run) -> str | None:
return None
if wrong.verdict == BIOS_ERROR:
return REFUSED
# The log names the BIOS the core ran: that settles it before any screen.
if wrong.verdict == BOOTS and wrong.bios_used == "builtin":
return REFUSED
if wrong.verdict == BOOTS and wrong.bios_used.startswith("staged:"):
return USED
if wrong.verdict == BOOTS:
if wrong.shows_bios:
return USED
+145 -7
View File
@@ -1007,7 +1007,7 @@ def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Varian
state_out.parent.mkdir(parents=True, exist_ok=True)
rep = run_once(args, system_dir, scratch, state_out, png_dir)
oracle = oracle_for(args.oracle_data, args.content_sha1, rep.get("library_name"))
rep["verdict"], rep["verdict_evidence"] = judge(rep, oracle)
judge_repetition(rep, oracle)
if rep.get("exit"):
flags.append(f"repetition {index}: {rep['verdict_evidence']['detail']}")
if not rep.get("loaded"):
@@ -1023,15 +1023,14 @@ def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Varian
if rep.get("observables") != reference:
differing = sorted(k for k in (reference or {}) if (rep.get("observables") or {}).get(k) != reference[k])
flags.append(f"repetition {index} disagrees with repetition 0 on {', '.join(differing)}")
verdict, evidence = variant_verdict(reps)
return {
out = {
"label": variant.label,
"system_files": {dest: sha for dest, sha in sorted(staged.items())},
"repetitions": reps,
"flags": flags,
"verdict": verdict,
"verdict_evidence": evidence,
}
judge_variant(out)
return out
# --------------------------------------------------------------------------
@@ -1111,6 +1110,16 @@ def build_parser() -> argparse.ArgumentParser:
check.add_argument("--claim", required=True, help="the statement under test")
check.add_argument("--method", required=True, help="how the run tests it")
check.add_argument("--repeat", type=int, default=2, help="runs of each variant, each in a fresh process")
rejudge = sub.add_parser("rejudge", help="judge recorded runs again against the current oracle, "
"offline: no core runs")
rejudge.add_argument("--record", metavar="DIR", default=str(ROOT),
help="the checkout whose measurements/ are judged again (default: this one)")
rejudge.add_argument("--emulator", action="append", default=[],
help="only these emulators' records (default: every one)")
rejudge.add_argument("--oracle", default=None, metavar="FILE",
help="the oracle to judge against (default: tests/campaign/oracle.yml)")
rejudge.add_argument("--dry-run", action="store_true",
help="print what would be appended, write nothing")
worker = sub.add_parser("_worker", help="internal: one run in this process, started by the others")
worker.add_argument("spec", help="JSON file describing the run")
worker.add_argument("out", help="JSON file the result is written to")
@@ -1274,7 +1283,9 @@ def oracle_for(data: dict, content_sha1: str | None, library_name: str | None) -
screens = [{"sha1": s["sha1"], "shows": s["shows"]} for s in core.get("error_screens") or ()]
loaded = [p["pattern"] for p in core.get("bios_patterns") or ()]
return {"references": references, "error_patterns": patterns, "error_screens": screens,
"bios_patterns": loaded}
"bios_patterns": loaded,
"fallback_patterns": [p["pattern"] for p in core.get("fallback_patterns") or ()],
"load_patterns": [p["pattern"] for p in core.get("load_patterns") or ()]}
def sampled(rep: dict) -> list[tuple[int, str]]:
@@ -1355,6 +1366,52 @@ def _without_the_game(rep: dict, oracle: dict, frames: list[tuple[int, str]]) ->
"and no error screen"}
BUILTIN = "builtin"
SYSTEM_PLACE = "<system>/"
def bios_used(rep: dict, oracle: dict) -> tuple[str, str | None]:
"""Which BIOS the core ran, from its log, and the line that says so.
`staged:<dest>` when a load pattern names the file under the system
directory, `builtin` when a fallback pattern says the core went to its
own BIOS (OpenBIOS, HLE), `unknown` otherwise. The last such line wins:
a core that tries a file and then falls back logs both, in that order.
A file the core opened is no proof: Beetle PSX hashes every file of the
folder before it decides.
"""
found: tuple[str, str | None] = ("unknown", None)
for line in rep.get("logs") or ():
text = line["text"]
if any(re.search(pattern, text) for pattern in oracle.get("fallback_patterns") or ()):
found = (BUILTIN, text)
for pattern in oracle.get("load_patterns") or ():
match = re.search(pattern, text)
path = match.groupdict().get("path") if match else None
if path:
found = (f"staged:{path.removeprefix(SYSTEM_PLACE)}", text)
return found
def judge_repetition(rep: dict, oracle: dict) -> None:
"""The verdict, its evidence and the BIOS used, written into the repetition."""
verdict, evidence = judge(rep, oracle)
used, line = bios_used(rep, oracle)
if line:
evidence["bios_log"] = line
if verdict == BOOTS and used == BUILTIN:
evidence["detail"] += ", on the core's built-in BIOS"
rep["verdict"], rep["verdict_evidence"], rep["bios_used"] = verdict, evidence, used
def judge_variant(variant: dict) -> None:
"""The variant's verdict and BIOS: what every repetition agrees on."""
reps = variant["repetitions"]
variant["verdict"], variant["verdict_evidence"] = variant_verdict(reps)
used = {rep.get("bios_used", "unknown") for rep in reps}
variant["bios_used"] = used.pop() if len(used) == 1 else "unknown"
def variant_verdict(reps: Sequence[dict]) -> tuple[str, dict]:
"""The verdict all repetitions agree on, else unknown."""
verdicts = [rep["verdict"] for rep in reps]
@@ -1363,7 +1420,7 @@ def variant_verdict(reps: Sequence[dict]) -> tuple[str, dict]:
evidence = {"rule": "repetitions_agree",
"detail": f"{len(reps)} of {len(reps)} repetitions: {first['detail']}",
"repetitions": verdicts}
for key in ("frame", "sha1", "stages", "pattern", "log", "exit_status", "signal"):
for key in ("frame", "sha1", "stages", "pattern", "log", "exit_status", "signal", "bios_log"):
if key in first:
evidence[key] = first[key]
return verdicts[0], evidence
@@ -1535,8 +1592,89 @@ def _ensure(path: Path) -> Path:
return path
def rejudged(record: dict, data: dict, identity: dict | None, now: str) -> dict | None:
"""The record judged again from its own runs, superseding it, or None
when the judgement it carries is the one the oracle gives now.
Nothing of the runs changes: logs, frames, files and timings are the
record's. The verdicts, their evidence, bios_used and inputs.oracle are
computed again, and judged_at and judged_by say when and by what.
"""
import copy # noqa: PLC0415
# Another probe (standalone_probe, pack_probe) judges with its own rules.
if record.get("record_version", 0) < 4 or record["frontend"].get("script") != "scripts/core_probe.py":
return None
out = copy.deepcopy(record)
content_sha1 = ((out["inputs"].get("content") or {}).get("sha1"))
for variant in out["variants"]:
for rep in variant["repetitions"]:
judge_repetition(rep, oracle_for(data, content_sha1, rep.get("library_name")))
judge_variant(variant)
first = out["variants"][0]["repetitions"][0]
out["inputs"]["oracle"] = ({**identity, **oracle_for(data, content_sha1, first.get("library_name"))}
if identity else None)
if out.get("check"):
out["check"]["result"]["verdicts"] = {v["label"]: v["verdict"] for v in out["variants"]}
def judgement(item: dict) -> list:
return [[v.get("verdict"), v.get("verdict_evidence"), v.get("bios_used"),
[[r.get("verdict"), r.get("verdict_evidence"), r.get("bios_used")] for r in v["repetitions"]]]
for v in item["variants"]]
if record.get("record_version") == RECORD_VERSION_NOW() and judgement(out) == judgement(record):
return None
for key in ("id", "migrated_from_version"):
out.pop(key, None)
out["record_version"] = RECORD_VERSION_NOW()
out["supersedes"] = record["id"]
out["judged_at"] = now
ident = frontend_identity()
out["judged_by"] = {"script": ident["script"], "commit": ident["commit"],
"script_sha256": ident["script_sha256"]}
return out
def RECORD_VERSION_NOW() -> int: # noqa: N802
from measurement_records import RECORD_VERSION # noqa: PLC0415
return RECORD_VERSION
def rejudge_records(args: argparse.Namespace) -> int:
"""Append, for each live record whose judgement the oracle now gives
otherwise, the record judged again, superseding it."""
from measurement_records import line_text, live, read_records, records_path, seal # noqa: PLC0415
root = Path(args.record)
data, identity = load_oracle(Path(args.oracle) if args.oracle else DEFAULT_ORACLE)
now = dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
stored = read_records(root)
alive = {r["id"] for r in live(s.record for s in stored)}
written = 0
for entry in stored:
if entry.record["id"] not in alive or (args.emulator and entry.emulator not in args.emulator):
continue
new = rejudged(entry.record, data, identity, now)
if new is None:
continue
sealed = seal(new)
before = {v["label"]: v.get("verdict") for v in entry.record["variants"]}
after = {v["label"]: f"{v['verdict']} ({v['bios_used']})" for v in sealed["variants"]}
print(json.dumps({"emulator": entry.emulator, "supersedes": entry.record["id"], "id": sealed["id"],
"before": before, "after": after}))
written += 1
if not args.dry_run:
with (root / records_path(entry.emulator)).open("a", encoding="utf-8") as handle:
handle.write(line_text(sealed) + "\n")
print(f"{written} records judged again" + (" (dry run, nothing written)" if args.dry_run else ""),
file=sys.stderr)
return 0
def main(argv: Sequence[str] | None = None) -> int:
args = build_parser().parse_args(argv)
if args.mode == "rejudge":
return rejudge_records(args)
if args.mode == "_worker":
spec = json.loads(Path(args.spec).read_text(encoding="utf-8"))
Path(args.out).write_text(json.dumps(run_worker(spec)), encoding="utf-8")
+4 -46
View File
@@ -2455,7 +2455,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
"cross-reference.md": "Every emulator core each retrogaming platform runs, the systems it emulates and the upstream project it comes from.",
"gaps.md": "Current RetroBIOS verification gaps, platform-to-emulator divergences, missing files and documented source limitations.",
"exports.md": "Each platform's own BIOS list (System.dat, batocera-systems, es_bios.xml and the rest) with the hashes and files the emulator source code shows, offered to its maintainers.",
sitemeasurements.EXPLORER_PAGE: "Emulator cores run headless in real conditions: each test with its claim, result, method, computer and full record, filterable, and as SQLite.",
sitemeasurements.EXPLORER_PAGE: "The records of emulator cores run headless on demand, each with its claim, verdict, the BIOS it ran, method, computer and full record, filterable, and as SQLite.",
"provenance.md": "Hash-based comparison of RetroBIOS metadata with No-Intro, Redump and TOSEC catalog snapshots.",
"explore.md": "Systems, emulators and installer options to explore, with the figures of the BIOS collection.",
"which-pack.md": "Pick your device or setup and get its BIOS files: the folder they go in, one command that installs and checks them, or a ZIP pack to extract.",
@@ -4332,9 +4332,6 @@ def generate_platform_page(
no_list = not cov["total"] and registry.get(name, {}).get("native_list") == "none"
if name in GUIDES:
lines += _platform_guide(name, display, cov, registry, no_list)
# Before the reference: the links below it are read under that heading.
lines.extend(sitemeasurements.pack_tested_section(_PACK_ITEMS.get(name, [])))
lines += [
"## Reference: every file by system { #reference-every-file-by-system }",
"",
@@ -7202,11 +7199,6 @@ def system_pages(
_section_label(manufacturer, console),
_install_pairs(installs.get(console, []), lead, consumers))
def region_block(console: str, level: int = 3) -> list[str]:
# Under the console's heading on a maker's page, a section of its
# own on the console's page.
return sitemeasurements.region_section(_REGION_RESULTS.get((slug, console), []), level)
def all_sections() -> dict[str, list[Card]]:
anchors: set[str] = set()
return {console: cards_of(console, anchors) for console in ordered}
@@ -7248,7 +7240,6 @@ def system_pages(
lines.append("")
lines.extend(start_of(console))
lines.extend(install_block(console))
lines.extend(region_block(console))
lines.extend(_cards_markdown(cards))
lines.append("")
lines.extend(_next_block([*_NEXT_FROM_SYSTEMS, json_next]))
@@ -7310,7 +7301,6 @@ def system_pages(
if console not in moved:
summary.extend(start_of(console))
summary.extend(install_block(console))
summary.extend(region_block(console))
summary.extend(_cards_markdown(cards_of(console, summary_anchors)))
summary.append("")
continue
@@ -7395,8 +7385,6 @@ def system_pages(
lines.extend(legend)
lines.extend(_romset_note({console: consoles[console]}))
lines.extend(install_block(console))
if index == 0:
lines.extend(region_block(console, 2))
lines.extend(["## Files", ""])
lines.extend(_cards_markdown(part))
lines.append("")
@@ -9125,13 +9113,8 @@ def _said_as_list(items: list[str]) -> str:
# Data directory members and revisions, read once per build.
_DATA_MEMBERS: dict[str, dict[str, dict]] = {}
_DATA_REVISIONS: dict[str, dict] = {}
# The measurement index items of each emulator, for its Tested block.
# The measurement index items of each emulator, for the proof lines of its page.
_MEASUREMENT_ITEMS: dict[str, list[dict]] = {}
# (manufacturer slug, console) -> the region outcomes of its systems
# (region_check.py), shown under the console on its system page.
_REGION_RESULTS: dict[tuple[str, str], list[dict]] = {}
# Pack probes by platform key, for the Tested block of a platform page.
_PACK_ITEMS: dict[str, list[dict]] = {}
# The emulator pages that list each data directory, by ref.
_DATA_READERS: dict[str, list[tuple[str, str]]] = {}
@@ -9358,19 +9341,6 @@ def _summary_table(files: list[dict], indexes: list[int], cell: Callable[[int, s
]
def _tested_file_hrefs(name: str, files: list[dict]) -> dict[str, str]:
"""The card each file name a test varies leads to: its name, aliases
and path, folded to lower case, the first card of a name winning."""
hrefs: dict[str, str] = {}
for f, anchor in zip(files, emulator_file_anchors(files)):
page = emulator_card_page(name, anchor)
href = f"#{anchor}" if page == name else markup.site_href(f"emulators/{page}/") + f"#{anchor}"
for label in (f.get("name"), *(f.get("aliases") or ()), f.get("path")):
if label:
hrefs.setdefault(Path(str(label)).name.casefold(), href)
return hrefs
def plan_emulator_pages(profiles: dict) -> None:
"""Record which page holds each emulator card, before any page links one."""
stems = set()
@@ -9762,8 +9732,7 @@ def emulator_pages(
[files[i] for i in entries],
[rows[i] for i in entries] if table_form else None)
lines.extend(sitemeasurements.tested_section(
name, _MEASUREMENT_ITEMS.get(name, []), _tested_file_hrefs(name, files)))
lines.extend(sitemeasurements.proof_section(_MEASUREMENT_ITEMS.get(name, [])))
blocks = _render_structured_blocks(profile)
if blocks:
@@ -11862,13 +11831,6 @@ def _generate(args: argparse.Namespace) -> None:
_system_pages_from_data(manufacturers, coverages, db, profiles)
)
print(f" {len(_system_page_map)} system IDs mapped to pages")
from siterender import _page_by_norm # noqa: PLC0415
_REGION_RESULTS.clear()
for result in sitemeasurements.site_region_results(repo=REPO_URL):
place = _system_page_map.get(result["system"]) or _page_by_norm(result["system"])
if place:
_REGION_RESULTS.setdefault(place, []).append(result)
set_console_sections(manufacturers, program_names(profiles))
from siterender import link_coverage, system_ids # noqa: PLC0415
@@ -11921,13 +11883,9 @@ def _generate(args: argparse.Namespace) -> None:
_record(path)
_MEASUREMENT_ITEMS.clear()
_MEASUREMENT_ITEMS.update(sitemeasurements.by_emulator(measured.index))
_PACK_ITEMS.clear()
_PACK_ITEMS.update(sitemeasurements.by_platform(measured.index))
print(f" {measured.index['records']} records, {measured.index['count']} tests")
write_if_changed(str(docs / sitemeasurements.EXPLORER_PAGE),
sitemeasurements.generate_explorer_page(
measured.index, {sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))})
+ reference_next())
sitemeasurements.generate_explorer_page(measured.index) + reference_next())
print("Generating static API and bulk data exports...")
exports = generate_data_exports(
+3 -2
View File
@@ -27,9 +27,10 @@ from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
RECORD_VERSION = 4
RECORD_VERSION = 5
# 3: before the boot oracle. 4: every repetition and variant carries a verdict.
RECORD_VERSIONS = (3, 4)
# 5: and the BIOS the core ran.
RECORD_VERSIONS = (3, 4, 5)
RECORDS_DIR = "measurements"
ID_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$")
SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:[-_][a-z0-9]+)*$")
+31 -6
View File
@@ -144,7 +144,8 @@ def outcome(records: dict[str, dict], entries: list[dict], region: str) -> dict:
continue
loaded = all(rep.get("loaded") for v in record["variants"] for rep in v["repetitions"])
item = {"record": record["id"], "verdict": verdict(record), "loaded": loaded,
"staged": sorted(record["variants"][0]["system_files"])}
"staged": sorted(record["variants"][0]["system_files"]),
"bios_used": record["variants"][0].get("bios_used", "unknown")}
if full and label != "full":
item["as_full"] = loaded and frames(record) == frames(full)
results[label] = item
@@ -173,22 +174,46 @@ def outcome(records: dict[str, dict], entries: list[dict], region: str) -> dict:
"own_boots": boots("own"),
"others_boot": boots("others"),
"pack_boots": boots("pack"),
# The BIOS each ran, from its log: a boot is not the staged file's
# until the log names it (core_probe.bios_used).
"own_on": (results.get("own") or {}).get("bios_used", "unknown"),
"others_on": (results.get("others") or {}).get("bios_used", "unknown"),
"judged_by": "verdict" if full_verdict not in (None, "unknown") else "full_run_frames",
}
BUILTIN = "builtin"
def staged_name(used: str | None) -> str | None:
"""The file a bios_used names, or None."""
return used.removeprefix("staged:") if used and used.startswith("staged:") else None
def sentence(game_region: str, result: dict) -> str:
"""The outcome in words, for the site."""
"""The outcome in words, for the site. A boot is said to be another
region's BIOS only when the core's log names that file: a core that
refuses the staged files and runs its own BIOS (OpenBIOS, HLE) boots
too, on that built-in BIOS."""
word = REGION_WORDS.get(game_region, game_region)
own, others = result.get("own_boots"), result.get("others_boot")
if own is None:
return f"The {word} game has no run with its own BIOS yet."
first = f"The {word} BIOS {'boots' if own else 'does not boot'} the {word} game"
if own and result.get("own_on") == BUILTIN:
first = f"The {word} game boots on the core's built-in BIOS, its own BIOS file was refused"
else:
first = f"The {word} BIOS {'boots' if own else 'does not boot'} the {word} game"
if others is None:
return first + ". The other regions' BIOS have no verdict yet."
if others:
return first + ", and so do the other regions' BIOS."
return first + ", the other regions' BIOS do not."
if not others:
return first + ", the other regions' BIOS do not."
used = result.get("others_on")
if used == BUILTIN:
return first + ", and on its built-in BIOS with the others' files refused."
name = staged_name(used)
if name:
return first + f", and so does {name}, another region's BIOS."
return first + ", and with the others' files on a BIOS its log does not name."
# --------------------------------------------------------------------------
File diff suppressed because it is too large. Load diff