mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 15:43:45 -06:00
Abdessamad Derraz
9c07dabfae
Records that back the same field or issue claim share one line. Each line says what the runs showed: the BIOS the core ran, the file strace saw opened, or what a maintainer watched in an issue's run (tests/campaign/observations.yml). A run with nothing to state stays off the page.
756 lines
34 KiB
Python
756 lines
34 KiB
Python
"""The measurements the site shows, from the records under measurements/.
|
|
|
|
The records are core probe runs, one append-only .jsonl per emulator with one
|
|
full record per line (measurement_records.py). From them the build writes,
|
|
never committed:
|
|
|
|
public/data/measurements/index.json the compact index the explorer and
|
|
the emulator pages read, one item
|
|
per emulator and subject
|
|
public/downloads/measurements.sqlite every record, normalised
|
|
public/downloads/measurements/<emu>.sqlite one database per emulator
|
|
|
|
Three places read the index: the Proof lines of an emulator page (only the
|
|
records a profile's `evidence:` names, or one tied to an issue), the
|
|
Measurements explorer page and the host table of the methodology page. Runs
|
|
are made on demand, to settle a dispute or an issue, to back a correction
|
|
sent to a platform, or to smoke-test packs before a release: there is no
|
|
showcase of every run. A record links its line at the last commit that
|
|
touched measurements/. Benchmarks stay in the records and the SQLite files,
|
|
never on a page.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
import measurement_records as mr
|
|
import sitemarkup as markup
|
|
from artifacts import json_text, write_if_changed
|
|
from siterender import _escape_tags
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
REPO = "https://github.com/Abdess/retrobios"
|
|
EXPLORER_PAGE = "measurements.md"
|
|
METHOD_PAGE = "wiki/measurements.md"
|
|
HOSTS_MARKER = "<!-- measurements-hosts -->"
|
|
INDEX_PATH = Path("data") / "measurements" / "index.json"
|
|
SQLITE_PATH = Path("downloads") / "measurements.sqlite"
|
|
PER_EMULATOR_DIR = Path("downloads") / "measurements"
|
|
SQLITE_SCHEMA_VERSION = "3"
|
|
# Observables the variants table leaves out: one value per sampled frame.
|
|
BULKY_OBSERVABLES = ("frame_hashes",)
|
|
INDEX_SCHEMA_VERSION = 1
|
|
# What a maintainer saw in an issue's runs, by record subject.
|
|
OBSERVATIONS = Path("tests") / "campaign" / "observations.yml"
|
|
# A Proof line lists this many record links at most, else one link to the file.
|
|
MAX_RECORD_LINKS = 8
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Index
|
|
# --------------------------------------------------------------------------
|
|
|
|
def host_summary(host: dict) -> str:
|
|
parts = [host.get("cpu_model") or "unknown CPU"]
|
|
if host.get("cpu_count"):
|
|
parts.append(f"{host['cpu_count']} CPUs")
|
|
parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip()
|
|
if "Linux" in host.get("os", "") else host.get("os", ""))
|
|
parts.append(f"Python {host.get('python', '')}")
|
|
before, after = host.get("load_before"), host.get("load_after")
|
|
if before and after:
|
|
parts.append(f"load {before[0]} before, {after[0]} after")
|
|
return ", ".join(part for part in parts if part)
|
|
|
|
|
|
def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str:
|
|
return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
|
|
|
|
|
|
BOOTS_BUILTIN = "boots on built-in BIOS"
|
|
|
|
|
|
def shown_verdict(variant: dict) -> str:
|
|
"""A variant's verdict as the pages say it: a boot on the core's
|
|
built-in BIOS is named so, never taken for the staged file's."""
|
|
if variant["verdict"] == "boots" and variant.get("bios_used") == "builtin":
|
|
return BOOTS_BUILTIN
|
|
return variant["verdict"]
|
|
|
|
|
|
def verdicts_of(record: dict) -> list[dict]:
|
|
"""Each variant's boot verdict, from record version 4 on, and the BIOS
|
|
it ran from version 5 on."""
|
|
return [{"variant": v["label"], "verdict": shown_verdict(v), "bios_used": v.get("bios_used", "unknown"),
|
|
"detail": (v.get("verdict_evidence") or {}).get("detail", ""),
|
|
"staged": sorted(name for name, sha1 in (v.get("system_files") or {}).items() if sha1)}
|
|
for v in record["variants"] if "verdict" in v]
|
|
|
|
|
|
def result_of(record: dict) -> tuple[str, str]:
|
|
"""The status and statement a page shows for a record.
|
|
|
|
A run, or a check compared on the sampled frames, is shown by its boot
|
|
verdicts: one verdict when every variant has it, else `mixed`, each
|
|
variant named in the statement. A check on another observable keeps
|
|
its comparison. A frame comparison recorded before the boot oracle
|
|
shows `no verdict`: frames that differ prove nothing about a boot.
|
|
"""
|
|
check = record.get("check") or {}
|
|
result = check.get("result") or {}
|
|
verdicts = verdicts_of(record)
|
|
by_frames = record["kind"] == "run" or check.get("observable") == "frame_hashes"
|
|
if verdicts and by_frames:
|
|
values = {v["verdict"] for v in verdicts}
|
|
status = values.pop() if len(values) == 1 else "mixed"
|
|
if record["kind"] == "run":
|
|
return status, verdicts[0]["detail"]
|
|
return status, ". ".join(f"{v['variant']}: {v['verdict']}" for v in verdicts)
|
|
if by_frames:
|
|
return "no verdict", ("recorded before the boot oracle. Its frames were compared, which "
|
|
"settles nothing about a boot")
|
|
return result.get("status") or "ran", result.get("statement") or ""
|
|
|
|
|
|
def opened(record: dict) -> str | None:
|
|
"""The staged name a check varied, when strace saw the core open it in
|
|
every repetition of a variant that staged it."""
|
|
name = (record.get("check") or {}).get("variable")
|
|
if not isinstance(name, str):
|
|
return None
|
|
for variant in record["variants"]:
|
|
reps = variant.get("repetitions") or []
|
|
if (variant.get("system_files") or {}).get(name) and reps and all(
|
|
any(f.get("path") == name for f in rep.get("files_opened") or ()) for rep in reps):
|
|
return name
|
|
return None
|
|
|
|
|
|
def load_observations(path: Path) -> dict[str, dict]:
|
|
"""Each record id or subject an observation covers, with its issue
|
|
number and words."""
|
|
import yaml # noqa: PLC0415
|
|
|
|
if not path.is_file():
|
|
return {}
|
|
out = {}
|
|
for issue, entries in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).items():
|
|
number = ISSUE_SUBJECT.match(f"{issue}-")["number"]
|
|
for entry in entries:
|
|
for key in [*entry.get("records", ()), *entry.get("subjects", ())]:
|
|
out[key] = {"issue": number, "said": entry["said"]}
|
|
return out
|
|
|
|
|
|
def shown_of(record: dict) -> list | None:
|
|
"""What a record showed that a page can state: ["text", words] from
|
|
its boot verdicts, ["opened", name] for a staged name strace saw the
|
|
core open, or None."""
|
|
verdicts = verdicts_of(record)
|
|
if verdicts:
|
|
return ["text", proof_result(verdicts)]
|
|
name = opened(record)
|
|
return ["opened", name] if name else None
|
|
|
|
|
|
def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
|
|
system_names: dict[str, str], ancestry: set[str] | None = None,
|
|
records: list[dict] | None = None) -> dict:
|
|
record = entry.record
|
|
profile = profiles.get(entry.emulator) or {}
|
|
check = record.get("check") or {}
|
|
core = record["core"]
|
|
content = (record["inputs"].get("content") or {})
|
|
return {
|
|
"emulator": entry.emulator,
|
|
"emulator_name": str(profile.get("emulator", entry.emulator)),
|
|
"profiled": bool(profile),
|
|
"systems": sorted(set(profile.get("systems") or [])),
|
|
"system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))],
|
|
"subject": record["subject"],
|
|
"kind": record["kind"],
|
|
"claim": check.get("claim"),
|
|
"method": check.get("method"),
|
|
"observable": check.get("observable"),
|
|
"variable": check.get("variable"),
|
|
"status": result_of(record)[0],
|
|
"statement": result_of(record)[1],
|
|
"verdicts": verdicts_of(record),
|
|
"content": content.get("label"),
|
|
"variants": len(record["variants"]),
|
|
"repetitions": record["inputs"]["repetitions"],
|
|
"flags": len(record["flags"]),
|
|
"date": record["recorded_at"],
|
|
"host": host_summary(record["host"]),
|
|
"core": {
|
|
"library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])),
|
|
"file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"),
|
|
"commit": core.get("commit"), "build": core.get("build_command"),
|
|
},
|
|
"frontend_commit": record["frontend"].get("commit"),
|
|
"id": record["id"],
|
|
"runs": runs,
|
|
"record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line),
|
|
"records": records if records is not None else [],
|
|
"sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
|
|
"proves": proves(ancestry or {record["id"]}, profile),
|
|
}
|
|
|
|
|
|
def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
|
|
system_names: dict[str, str] | None = None,
|
|
observations: dict[str, dict] | None = None) -> dict:
|
|
"""One item per (emulator, subject): its latest record not superseded."""
|
|
by_subject: dict[tuple[str, str], list[mr.Stored]] = {}
|
|
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
|
|
ancestors = lineage(stored)
|
|
for entry in stored:
|
|
# A benchmark only chooses between variants: it stays in the records.
|
|
if entry.record["id"] in live_ids and entry.record["kind"] != "bench":
|
|
by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry)
|
|
items = []
|
|
for key in sorted(by_subject):
|
|
entries = by_subject[key]
|
|
latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
|
|
profile = profiles.get(key[0]) or {}
|
|
records = [{"url": permalink((source or {}).get("repo"), (source or {}).get("commit"),
|
|
mr.records_path(s.emulator), s.line),
|
|
"proves": proves(ancestors.get(s.record["id"]) or {s.record["id"]}, profile),
|
|
"shown": shown_of(s.record),
|
|
"observed": (observations or {}).get(s.record["id"]) or (observations or {}).get(key[1])}
|
|
for s in sorted(entries, key=lambda s: s.line)]
|
|
items.append(index_item(latest, len(entries), source or {}, profiles, system_names or {},
|
|
ancestors.get(latest.record["id"]), records))
|
|
items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
|
|
return {
|
|
"schema_version": INDEX_SCHEMA_VERSION,
|
|
"kind": "measurements",
|
|
"repo": (source or {}).get("repo"),
|
|
"commit": (source or {}).get("commit"),
|
|
"records": len(stored),
|
|
"count": len(items),
|
|
"items": items,
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# SQLite
|
|
# --------------------------------------------------------------------------
|
|
|
|
SCHEMA_SQL = """
|
|
PRAGMA journal_mode = OFF;
|
|
PRAGMA synchronous = OFF;
|
|
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
CREATE TABLE cores (
|
|
sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT,
|
|
build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT
|
|
);
|
|
CREATE TABLE hosts (
|
|
id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL,
|
|
kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL
|
|
);
|
|
CREATE TABLE emulator_systems (
|
|
emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system)
|
|
);
|
|
CREATE TABLE runs (
|
|
id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL,
|
|
kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT,
|
|
superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
|
|
host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
|
|
max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
|
|
script_modified INTEGER, flags TEXT NOT NULL, record_url TEXT NOT NULL,
|
|
line INTEGER NOT NULL,
|
|
migrated_from_version INTEGER
|
|
);
|
|
CREATE TABLE inputs (
|
|
run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT,
|
|
content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL,
|
|
warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL,
|
|
observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL
|
|
);
|
|
CREATE TABLE content_tracks (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
|
|
file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL,
|
|
PRIMARY KEY (run_id, position)
|
|
);
|
|
CREATE TABLE checks (
|
|
run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL,
|
|
observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL,
|
|
statement TEXT NOT NULL
|
|
);
|
|
CREATE TABLE check_groups (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
|
|
value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position)
|
|
);
|
|
CREATE TABLE variants (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
|
|
label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
|
|
flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
|
|
files_probed_absent TEXT, tracing TEXT, verdict TEXT, verdict_evidence TEXT,
|
|
PRIMARY KEY (run_id, position)
|
|
);
|
|
CREATE TABLE system_files (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
|
|
destination TEXT NOT NULL, sha1 TEXT,
|
|
PRIMARY KEY (run_id, variant_position, destination)
|
|
);
|
|
CREATE TABLE benchmarks (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL,
|
|
median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL,
|
|
min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric)
|
|
);
|
|
CREATE INDEX runs_emulator_idx ON runs(emulator, subject);
|
|
CREATE INDEX runs_subject_idx ON runs(subject);
|
|
CREATE INDEX emulator_systems_system_idx ON emulator_systems(system);
|
|
CREATE INDEX checks_claim_idx ON checks(claim);
|
|
CREATE INDEX system_files_sha1_idx ON system_files(sha1);
|
|
"""
|
|
|
|
TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks",
|
|
"checks", "check_groups", "variants", "system_files", "benchmarks")
|
|
|
|
|
|
def _host_key(host: dict) -> tuple:
|
|
return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel",
|
|
"machine", "python"))
|
|
|
|
|
|
def write_sqlite(destination: Path, stored: list[mr.Stored], source: dict | None, profiles: dict,
|
|
scratch: Path) -> None:
|
|
"""A deterministic database of the records: sorted rows, no clock."""
|
|
scratch.mkdir(parents=True, exist_ok=True)
|
|
temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite"
|
|
temp.unlink(missing_ok=True)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
stored = sorted(stored, key=lambda s: (s.emulator, s.line))
|
|
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
|
|
hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)}
|
|
connection = sqlite3.connect(temp)
|
|
try:
|
|
connection.executescript(SCHEMA_SQL)
|
|
meta = {
|
|
"schema_version": SQLITE_SCHEMA_VERSION,
|
|
"record_version": str(mr.RECORD_VERSION),
|
|
"measurements_repo": (source or {}).get("repo") or "",
|
|
"measurements_commit": (source or {}).get("commit") or "",
|
|
"license": "MIT",
|
|
"scope": "core probe records, one per line of measurements/<emulator>.jsonl",
|
|
}
|
|
connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
|
|
cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
|
|
connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [
|
|
(sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")),
|
|
json_text(core.get("build_command")), json_text(core.get("compiler")),
|
|
json_text(core.get("library_name")), json_text(core.get("library_version")))
|
|
for sha, core in sorted(cores.items())])
|
|
connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [
|
|
(number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])])
|
|
emulators = sorted({s.emulator for s in stored})
|
|
connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [
|
|
(emu, system) for emu in emulators
|
|
for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))])
|
|
for entry in stored:
|
|
_insert_run(connection, entry, source or {}, hosts, entry.record["id"] not in live_ids)
|
|
connection.commit()
|
|
connection.execute("VACUUM")
|
|
finally:
|
|
connection.close()
|
|
os.replace(temp, destination)
|
|
|
|
|
|
def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
|
|
hosts: dict, superseded: bool) -> None:
|
|
r = entry.record
|
|
host, front, inputs = r["host"], r["frontend"], r["inputs"]
|
|
rid = r["id"]
|
|
connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
|
|
int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
|
|
json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
|
|
front.get("commit"), front["script_sha256"],
|
|
None if front.get("script_modified") is None else int(front["script_modified"]),
|
|
json_text(r["flags"]),
|
|
permalink(source.get("repo", ""), source.get("commit", ""), mr.records_path(entry.emulator), entry.line),
|
|
entry.line, r.get("migrated_from_version")))
|
|
content = inputs.get("content") or {}
|
|
connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, content.get("label"), content.get("sha1"), content.get("size"),
|
|
json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"],
|
|
inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]),
|
|
inputs["frame_hash_every"], inputs["joypad_ports"]))
|
|
connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [
|
|
(rid, position, track["file"], track["sha1"], track["size"])
|
|
for position, track in enumerate(content.get("tracks") or [])])
|
|
check = r.get("check")
|
|
if check:
|
|
result = check["result"]
|
|
connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, check["claim"], check["method"], check["observable"], check["variable"],
|
|
result["status"], result["statement"]))
|
|
connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
|
|
(rid, position, json_text(group["value"]),
|
|
mr.value_sha1(group["value"]) if isinstance(group["value"], (list, dict)) else None,
|
|
json_text(group["variants"]))
|
|
for position, group in enumerate(result["groups"])])
|
|
for position, variant in enumerate(r["variants"]):
|
|
reps = variant["repetitions"]
|
|
first = reps[0]
|
|
# The first repetition stands for the variant, its per-frame hashes left out.
|
|
observables = {key: value for key, value in (first.get("observables") or {}).items()
|
|
if key not in BULKY_OBSERVABLES}
|
|
connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)),
|
|
json_text(variant["flags"]), json_text(observables),
|
|
json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")),
|
|
json_text(first.get("tracing")), variant.get("verdict"),
|
|
json_text(variant.get("verdict_evidence")) if "verdict_evidence" in variant else None))
|
|
connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
|
|
(rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
|
|
for metric, stats in sorted((r.get("timing") or {}).items()):
|
|
connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"]))
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Building the site's files
|
|
# --------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class Built:
|
|
index: dict
|
|
paths: list[Path]
|
|
|
|
|
|
def load_profiles(emulators_dir: Path) -> dict:
|
|
from common import load_emulator_profiles # noqa: PLC0415
|
|
|
|
return load_emulator_profiles(str(emulators_dir), skip_aliases=False)
|
|
|
|
|
|
def records_commit(root: Path) -> str | None:
|
|
"""The commit the record links name, with a warning when the records on
|
|
disk are not the ones it holds."""
|
|
commit = mr.last_commit(root)
|
|
if commit is None:
|
|
print(f"WARNING: no commit holds {root / mr.RECORDS_DIR}: the record links "
|
|
"lead nowhere until the records are committed.", file=sys.stderr)
|
|
return None
|
|
on_disk = {path: file.read_bytes() for path, file in mr.tree_files(root).items()}
|
|
if mr.committed_files(root, commit) != on_disk:
|
|
print(f"WARNING: {mr.RECORDS_DIR}/ differs from {commit[:12]}, the last commit that "
|
|
"touched it: links to records added since lead nowhere until they are "
|
|
"committed.", file=sys.stderr)
|
|
return commit
|
|
|
|
|
|
def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulators",
|
|
scratch: Path = ROOT / "tmp" / "site", system_names: dict[str, str] | None = None,
|
|
repo: str = REPO, commit: str | None = None) -> Built:
|
|
"""Write the index and the databases under the site's public folder.
|
|
|
|
``commit`` is the revision the record links name, by default the last
|
|
commit that touched the records.
|
|
"""
|
|
stored = mr.read_records(root)
|
|
source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)}
|
|
profiles = load_profiles(emulators_dir)
|
|
index = build_index(stored, source, profiles, system_names, load_observations(root / OBSERVATIONS))
|
|
paths = [public / INDEX_PATH]
|
|
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
|
|
write_if_changed(str(public / INDEX_PATH),
|
|
json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
|
|
write_sqlite(public / SQLITE_PATH, stored, source, profiles, scratch)
|
|
paths.append(public / SQLITE_PATH)
|
|
for emulator in sorted({s.emulator for s in stored}):
|
|
target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite"
|
|
write_sqlite(target, [s for s in stored if s.emulator == emulator], source, profiles, scratch)
|
|
paths.append(target)
|
|
return Built(index, paths)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Pages
|
|
# --------------------------------------------------------------------------
|
|
|
|
def by_emulator(index: dict) -> dict[str, list[dict]]:
|
|
"""Each emulator's items. A pack probe's run proves a pack, not the
|
|
emulator's page."""
|
|
out: dict[str, list[dict]] = {}
|
|
for item in index.get("items", []):
|
|
if not item["subject"].startswith(mr.PACK_SUBJECT_PREFIX):
|
|
out.setdefault(item["emulator"], []).append(item)
|
|
return out
|
|
|
|
|
|
ISSUE_SUBJECT = re.compile(r"^issue(?P<number>\d+)-")
|
|
|
|
|
|
VERDICT_WORDS = {"boots": "boots", "bios_error": "shows a BIOS error", "crash": "crashes",
|
|
"hang": "hangs", "unknown": "gives no verdict"}
|
|
|
|
|
|
def proof_result(verdicts: list[dict]) -> str:
|
|
"""What a record showed, in words, naming the BIOS each boot ran: a
|
|
boot on the core's built-in BIOS is never the staged file's. A single
|
|
run says what was staged, a check names each variant."""
|
|
def said(verdict: dict) -> str:
|
|
used = verdict.get("bios_used") or "unknown"
|
|
if verdict["verdict"] == BOOTS_BUILTIN:
|
|
return "boots on the core's built-in BIOS"
|
|
if verdict["verdict"] == "boots" and used.startswith("staged:"):
|
|
return f"boots on {used.removeprefix('staged:')}"
|
|
return VERDICT_WORDS.get(verdict["verdict"], verdict["verdict"].replace("_", " "))
|
|
if len(verdicts) == 1:
|
|
staged = verdicts[0].get("staged") or []
|
|
given = " and ".join(staged) if len(staged) < 3 else f"{', '.join(staged[:-1])} and {staged[-1]}"
|
|
return f"with {given or 'no BIOS'}, it {said(verdicts[0])}"
|
|
return ". ".join(f"{v['variant']}: {said(v)}" for v in verdicts)
|
|
|
|
|
|
def proof_claims(item: dict, record: dict) -> tuple[list[str], list | None]:
|
|
"""The claims one record of an item backs on its emulator's page, and
|
|
what it showed. An issue observation is the claim and needs no more.
|
|
No claim when the record shows nothing a page can state."""
|
|
observed = record.get("observed")
|
|
if observed:
|
|
return [f"issue {observed['issue']}: {observed['said']}"], None
|
|
shown = record.get("shown")
|
|
if shown is None:
|
|
return [], None
|
|
if record.get("proves"):
|
|
return record["proves"], shown
|
|
issue = ISSUE_SUBJECT.match(item["subject"])
|
|
if issue and shown[0] == "text":
|
|
return [f"issue {issue['number']}"], shown
|
|
return [], None
|
|
|
|
|
|
def _md(value: object) -> str:
|
|
"""Record text placed in a Markdown line: no span, link or tag out of it."""
|
|
return _cell(value).translate(_MARKDOWN)
|
|
|
|
|
|
def _records(urls: list[str]) -> str:
|
|
"""The record links of one statement, escaped and safe targets only."""
|
|
def place(url: str) -> tuple[str, int]:
|
|
path, _, line = url.partition("#L")
|
|
return path, int(line) if line.isdigit() else 0
|
|
targets = sorted(dict.fromkeys(t for u in urls if (t := _href(u))), key=place)
|
|
if not targets:
|
|
return ""
|
|
if len(targets) == 1:
|
|
return f" ([record]({targets[0]}))"
|
|
files = {t.split("#", 1)[0] for t in targets}
|
|
if len(targets) > MAX_RECORD_LINKS and len(files) == 1:
|
|
return f" ([{len(targets)} records]({files.pop()}))"
|
|
return " (records " + ", ".join(f"[{n}]({t})" for n, t in enumerate(targets, 1)) + ")"
|
|
|
|
|
|
def proof_line(claim: str, shown: list[tuple[list | None, list[str]]]) -> str:
|
|
"""One line per claim, its records collapsed: the files strace saw
|
|
opened in one list, each distinct result once with its records."""
|
|
names: dict[str, list[str]] = {}
|
|
texts: dict[str, list[str]] = {}
|
|
bare: list[str] = []
|
|
for what, urls in shown:
|
|
if what is None:
|
|
bare.extend(urls)
|
|
elif what[0] == "opened":
|
|
names.setdefault(what[1], []).extend(urls)
|
|
else:
|
|
texts.setdefault(what[1], []).extend(urls)
|
|
parts = []
|
|
if names:
|
|
parts.append("the core opened " + ", ".join(_md(n) for n in names)
|
|
+ _records([u for urls in names.values() for u in urls]))
|
|
parts.extend(_md(text) + _records(urls) for text, urls in texts.items())
|
|
if not parts:
|
|
return f"- {_md(claim)}{_records(bare)}."
|
|
return f"- {_md(claim)}: " + ". ".join(part[:1].upper() + part[1:] if n else part
|
|
for n, part in enumerate(parts)) + "."
|
|
|
|
|
|
def proof_section(items: list[dict]) -> list[str]:
|
|
"""The Proof lines of an emulator page: only the claims of the page a
|
|
run backs, one line per claim. Every other record stays on the
|
|
Measurements page."""
|
|
by_claim: dict[str, list] = {}
|
|
for item in sorted(items, key=lambda i: i["subject"]):
|
|
for record in item.get("records") or ():
|
|
claims, what = proof_claims(item, record)
|
|
for claim in claims:
|
|
by_claim.setdefault(claim, []).append((what, [record["url"]]))
|
|
if not by_claim:
|
|
return []
|
|
return ["## Proof", "",
|
|
f"Real runs that back this page ([how they are made](../../{METHOD_PAGE.removesuffix('.md')}/)):",
|
|
"", *(proof_line(claim, by_claim[claim]) for claim in sorted(by_claim)), ""]
|
|
|
|
|
|
def _text(value: object) -> str:
|
|
"""Record text as the page shows it. A semicolon joining two clauses
|
|
ends a sentence on the site: the record keeps the words as written."""
|
|
text = " ".join(str(value or "").split())
|
|
text = re.sub(r";\s+(\w)", lambda m: ". " + m[1].upper(), text)
|
|
return _escape_tags(text)
|
|
|
|
|
|
def _short(commit: str | None) -> str:
|
|
return f"`{commit[:12]}`" if commit else "unknown"
|
|
|
|
|
|
def proves(record_ids: set[str], profile: dict) -> list[str]:
|
|
"""The profile fields a record is evidence for, as `field of file`. A
|
|
record judged again stands for the ones it supersedes."""
|
|
out = []
|
|
for entry in profile.get("files") or ():
|
|
for name, ids in sorted((entry.get("evidence") or {}).items()):
|
|
if record_ids & set(ids):
|
|
out.append(f"{name} of {entry.get('name')}")
|
|
return out
|
|
|
|
|
|
def lineage(stored: list[mr.Stored]) -> dict[str, set[str]]:
|
|
"""Each record id with every id it supersedes, through the chain."""
|
|
parent = {s.record["id"]: s.record.get("supersedes") for s in stored}
|
|
out = {}
|
|
for rid in parent:
|
|
seen = {rid}
|
|
node = parent[rid]
|
|
while node and node not in seen:
|
|
seen.add(node)
|
|
node = parent.get(node)
|
|
out[rid] = seen
|
|
return out
|
|
|
|
|
|
# An MDX page reads a brace as the start of an expression, in text and in
|
|
# an attribute alike.
|
|
_BRACES = str.maketrans({"{": "{", "}": "}"})
|
|
|
|
|
|
def _attr(value: object) -> str:
|
|
return html.escape(" ".join(str(value or "").split()), quote=True).translate(_BRACES)
|
|
|
|
|
|
def _cell(value: object) -> str:
|
|
"""Record text placed in raw HTML: escaped as text, never read as
|
|
Markdown (a run's output is no markup), braces made inert for MDX. A
|
|
semicolon joining two clauses ends a sentence, as on the rest of the
|
|
site."""
|
|
text = " ".join(str(value or "").split())
|
|
text = re.sub(r";\s+(\w)", lambda m: ". " + m[1].upper(), text)
|
|
return html.escape(text, quote=True).translate(_BRACES)
|
|
|
|
|
|
# The proof line is Markdown: a character that starts a span, a link or
|
|
# an image in a run's output stays a character.
|
|
_MARKDOWN = str.maketrans({c: f"&#{ord(c)};" for c in "`*_[]()~!|\\"})
|
|
|
|
|
|
_SAFE_HREF = re.compile(r"^(?:https?://[^\s\"'<>]+|/[^\s\"'<>]*|\.{0,2}/?[\w.~-][^\s\"'<>:]*)$")
|
|
|
|
|
|
def _href(url: object) -> str | None:
|
|
"""A link target from a record: http(s) or relative only, else none."""
|
|
text = str(url or "")
|
|
return _attr(text) if text and _SAFE_HREF.match(text) and not text.lower().startswith(
|
|
("javascript:", "data:", "vbscript:")) else None
|
|
|
|
|
|
def hosts_table(index: dict) -> list[str]:
|
|
hosts: dict[str, int] = {}
|
|
for item in index.get("items", []):
|
|
# The load average differs per run: the computer is what comes before it.
|
|
computer = item["host"].split(", load ")[0]
|
|
hosts[computer] = hosts.get(computer, 0) + 1
|
|
if not hosts:
|
|
return ["No result is recorded yet.", ""]
|
|
return [
|
|
"| Computer | Tests |",
|
|
"|----------|------:|",
|
|
*(f"| {_text(host)} | {count} |" for host, count in sorted(hosts.items())),
|
|
"",
|
|
]
|
|
|
|
|
|
def with_hosts(page: str, index: dict) -> str:
|
|
"""The methodology page with the computers the results ran on."""
|
|
return page.replace(HOSTS_MARKER, "\n".join(hosts_table(index)).rstrip("\n"))
|
|
|
|
|
|
def generate_explorer_page(index: dict) -> str:
|
|
commit = index.get("commit")
|
|
repo = index.get("repo") or ""
|
|
source = (f"[`measurements/`]({repo}/tree/{commit}/{mr.RECORDS_DIR}) at {_short(commit)}"
|
|
if commit else f"`{mr.RECORDS_DIR}/` of this repository")
|
|
return "\n".join([
|
|
"# Measurements",
|
|
"",
|
|
f"Emulators run in real conditions: {index.get('records', 0)} records, "
|
|
f"{index.get('count', 0)} tests, read from {source}. "
|
|
f"[How they are made]({METHOD_PAGE}).",
|
|
"",
|
|
" ".join([
|
|
markup.button("SQLite of every record", "downloads/measurements.sqlite", primary=True),
|
|
markup.button("Index as JSON", "data/measurements/index.json"),
|
|
]),
|
|
"",
|
|
"Each row is the latest record of one test, with its verdict, the BIOS it ran, its "
|
|
"method and the computer it ran on. Runs are made on demand: to settle a dispute or an "
|
|
"issue, to back a correction sent to a platform, or to check the packs before a release. "
|
|
"One SQLite file per emulator sits under `downloads/measurements/`.",
|
|
"",
|
|
"<MeasurementsExplorer />",
|
|
"",
|
|
])
|
|
|
|
|
|
def sqlite_tables(path: Path) -> list[tuple[str, list[str]]]:
|
|
"""Each table of a database with its columns, in the order created."""
|
|
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
try:
|
|
names = [row[0] for row in connection.execute(
|
|
"SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")]
|
|
return [(name, [row[1] for row in connection.execute(f"PRAGMA table_info({name})")])
|
|
for name in names]
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def data_page_section(path: Path) -> list[str]:
|
|
"""The Data page's description of measurements.sqlite."""
|
|
return [
|
|
"## Measurements",
|
|
"",
|
|
"[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe "
|
|
f"record kept in [`{mr.RECORDS_DIR}/`]({REPO}/tree/main/{mr.RECORDS_DIR}), and "
|
|
"`downloads/measurements/<emulator>.sqlite` the records of one emulator, linked "
|
|
f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads "
|
|
"[`data/measurements/index.json`](data/measurements/index.json). A record line is "
|
|
"valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json).",
|
|
"",
|
|
"| Table | Columns |",
|
|
"|-------|---------|",
|
|
*(f"| `{name}` | {', '.join(f'`{c}`' for c in columns)} |" for name, columns in sqlite_tables(path)),
|
|
"",
|
|
"`meta` names the commit the record links point to and the record version. A list or object "
|
|
"value is JSON text, an absent one empty. `runs.superseded` is 1 for a record a "
|
|
"later one corrects.",
|
|
"",
|
|
"```sql",
|
|
"SELECT r.emulator, c.claim, c.status, r.recorded_at, h.cpu_model",
|
|
"FROM runs r JOIN checks c ON c.run_id = r.id JOIN hosts h ON h.id = r.host_id",
|
|
"WHERE r.superseded = 0 ORDER BY r.recorded_at;",
|
|
"```",
|
|
"",
|
|
]
|