mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 07:33:45 -06:00
Abdessamad Derraz
38824658bf
The Measurements page lists the latest probe of each platform, system, install path and core, and each platform page shows its own under Tested.
707 lines
32 KiB
Python
707 lines
32 KiB
Python
"""The measurements the site shows, from the records under measurements/.
|
|
|
|
The records are core probe runs, one append-only .jsonl per emulator with one
|
|
full record per line (measurement_records.py). From them the build writes,
|
|
never committed:
|
|
|
|
public/data/measurements/index.json the compact index the explorer and
|
|
the emulator pages read, one item
|
|
per emulator and subject
|
|
public/downloads/measurements.sqlite every record, normalised
|
|
public/downloads/measurements/<emu>.sqlite one database per emulator
|
|
|
|
Three places read the index: the Tested block of each emulator page, the
|
|
Measurements explorer page and the host table of the methodology page. A
|
|
number is never shown without its method and host beside it. A record links
|
|
its line at the last commit that touched measurements/.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
import measurement_records as mr
|
|
import sitemarkup as markup
|
|
from artifacts import json_text, write_if_changed
|
|
from siterender import _escape_tags
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
REPO = "https://github.com/Abdess/retrobios"
|
|
EXPLORER_PAGE = "measurements.md"
|
|
METHOD_PAGE = "wiki/measurements.md"
|
|
HOSTS_MARKER = "<!-- measurements-hosts -->"
|
|
INDEX_PATH = Path("data") / "measurements" / "index.json"
|
|
SQLITE_PATH = Path("downloads") / "measurements.sqlite"
|
|
PER_EMULATOR_DIR = Path("downloads") / "measurements"
|
|
SQLITE_SCHEMA_VERSION = "2"
|
|
# Observables the variants table leaves out: one value per sampled frame.
|
|
BULKY_OBSERVABLES = ("frame_hashes",)
|
|
INDEX_SCHEMA_VERSION = 1
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Index
|
|
# --------------------------------------------------------------------------
|
|
|
|
def host_summary(host: dict) -> str:
|
|
parts = [host.get("cpu_model") or "unknown CPU"]
|
|
if host.get("cpu_count"):
|
|
parts.append(f"{host['cpu_count']} CPUs")
|
|
parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip()
|
|
if "Linux" in host.get("os", "") else host.get("os", ""))
|
|
parts.append(f"Python {host.get('python', '')}")
|
|
before, after = host.get("load_before"), host.get("load_after")
|
|
if before and after:
|
|
parts.append(f"load {before[0]} before, {after[0]} after")
|
|
return ", ".join(part for part in parts if part)
|
|
|
|
|
|
def bench_statement(record: dict) -> str:
|
|
timing = record["timing"]
|
|
frame, fps = timing["ms_per_frame"], timing["fps"]
|
|
return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), "
|
|
f"min {frame['min']:.3f}, max {frame['max']:.3f}, over "
|
|
f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames")
|
|
|
|
|
|
def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str:
|
|
return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
|
|
|
|
|
|
def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
|
|
system_names: dict[str, str]) -> dict:
|
|
record = entry.record
|
|
profile = profiles.get(entry.emulator) or {}
|
|
check = record.get("check") or {}
|
|
core = record["core"]
|
|
content = (record["inputs"].get("content") or {})
|
|
return {
|
|
"emulator": entry.emulator,
|
|
"emulator_name": str(profile.get("emulator", entry.emulator)),
|
|
"profiled": bool(profile),
|
|
"systems": sorted(set(profile.get("systems") or [])),
|
|
"system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))],
|
|
"subject": record["subject"],
|
|
"kind": record["kind"],
|
|
"claim": check.get("claim"),
|
|
"method": check.get("method"),
|
|
"observable": check.get("observable"),
|
|
"variable": check.get("variable"),
|
|
"status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
|
|
"statement": (check.get("result") or {}).get("statement")
|
|
or (bench_statement(record) if record.get("timing") else ""),
|
|
"content": content.get("label"),
|
|
"variants": len(record["variants"]),
|
|
"repetitions": record["inputs"]["repetitions"],
|
|
"flags": len(record["flags"]),
|
|
"date": record["recorded_at"],
|
|
"host": host_summary(record["host"]),
|
|
"core": {
|
|
"library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])),
|
|
"file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"),
|
|
"commit": core.get("commit"), "build": core.get("build_command"),
|
|
},
|
|
"frontend_commit": record["frontend"].get("commit"),
|
|
"id": record["id"],
|
|
"runs": runs,
|
|
"record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line),
|
|
"sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
|
|
}
|
|
|
|
|
|
def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
|
|
system_names: dict[str, str] | None = None) -> dict:
|
|
"""One item per (emulator, subject): its latest record not superseded."""
|
|
by_subject: dict[tuple[str, str], list[mr.Stored]] = {}
|
|
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
|
|
for entry in stored:
|
|
if entry.record["id"] in live_ids:
|
|
by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry)
|
|
items = []
|
|
for key in sorted(by_subject):
|
|
entries = by_subject[key]
|
|
latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
|
|
items.append(index_item(latest, len(entries), source or {}, profiles, system_names or {}))
|
|
items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
|
|
return {
|
|
"schema_version": INDEX_SCHEMA_VERSION,
|
|
"kind": "measurements",
|
|
"repo": (source or {}).get("repo"),
|
|
"commit": (source or {}).get("commit"),
|
|
"records": len(stored),
|
|
"count": len(items),
|
|
"items": items,
|
|
}
|
|
|
|
|
|
def campaign_summary(root: Path, stored: list[mr.Stored], source: dict | None) -> dict:
|
|
"""The last campaign pass and the findings still waiting for review.
|
|
|
|
A finding's history is its lines under one key: the latest line is its
|
|
state, and only an open or flagged one is listed.
|
|
"""
|
|
runs = mr.read_campaign(root, mr.CAMPAIGN_RUNS)
|
|
latest: dict[str, dict] = {}
|
|
applied = 0
|
|
for line in mr.read_campaign(root, mr.FINDINGS):
|
|
latest[line["key"]] = line
|
|
applied += line["status"] == "applied"
|
|
where = {s.record["id"]: s for s in stored}
|
|
links = {}
|
|
for line in latest.values():
|
|
for rid in line["evidence"]:
|
|
found = where.get(rid)
|
|
if found and source and source.get("commit"):
|
|
links[rid] = permalink(source["repo"], source["commit"],
|
|
mr.records_path(found.emulator), found.line)
|
|
open_items = [{**line, "links": [links.get(rid) for rid in line["evidence"]]}
|
|
for _, line in sorted(latest.items()) if line["status"] in ("open", "flagged")]
|
|
return {"last_run": runs[-1] if runs else None, "passes": len(runs), "applied": applied,
|
|
"open": open_items}
|
|
|
|
|
|
def pack_summary(root: Path, source: dict | None) -> list[dict]:
|
|
"""The latest pack probe of each platform, install path, system, core
|
|
and game, with a link to its line."""
|
|
latest: dict[tuple, tuple[int, dict]] = {}
|
|
for number, line in enumerate(mr.read_campaign(root, mr.PACK_PROBES), 1):
|
|
key = (line["platform"], line["install"], line["system"], line["core"]["file"],
|
|
line["content"]["label"])
|
|
latest[key] = (number, line)
|
|
items = []
|
|
for key in sorted(latest):
|
|
number, line = latest[key]
|
|
url = (permalink(source["repo"], source["commit"], mr.records_path(mr.PACK_PROBES), number)
|
|
if source and source.get("commit") else None)
|
|
items.append({
|
|
"platform": line["platform"], "platform_version": line["platform_version"],
|
|
"install": line["install"], "system": line["system"], "emulator": line["emulator"],
|
|
"core": line["core"]["file"].removesuffix("_libretro.so"),
|
|
"content": line["content"]["label"], "verdict": line["verdict"],
|
|
"date": line["probed_at"], "opened": len(line["files_opened"]),
|
|
"findings": len(line["findings"]), "id": line["id"], "record_url": url,
|
|
})
|
|
return items
|
|
|
|
|
|
def by_platform(index: dict) -> dict[str, list[dict]]:
|
|
out: dict[str, list[dict]] = {}
|
|
for item in index.get("packs") or ():
|
|
out.setdefault(item["platform"], []).append(item)
|
|
return out
|
|
|
|
|
|
INSTALL_WORDS = {"pack": "pack", "installer": "installer"}
|
|
|
|
|
|
def _pack_row(item: dict, platform_column: bool) -> str:
|
|
link = f"[line]({item['record_url']})" if item["record_url"] else _code(item["id"])
|
|
cells = ([_plain(item["platform"])] if platform_column else []) + [
|
|
_plain(item["system"]), INSTALL_WORDS[item["install"]], _code(item["core"]),
|
|
_plain(item["content"]), f"`{item['verdict']}`", str(item["findings"]), link]
|
|
return "| " + " | ".join(cells) + " |"
|
|
|
|
|
|
PACKS_INTRO = ("A game started with the platform's BIOS folder as a user gets it, from the "
|
|
"pack or from the installer ([how]({prefix}{page}#packs)). A finding is a file "
|
|
"the core looked for that the folder lacks or holds under another path.")
|
|
PLATFORM_INTRO = "Games started with this BIOS folder as you install it ([how]({prefix}{page}#packs))."
|
|
|
|
|
|
def packs_section(items: list[dict], heading: str = "## Packs", platform_column: bool = True,
|
|
prefix: str = "", intro: str = PACKS_INTRO) -> list[str]:
|
|
"""Packs tested the way a user installs them: one row per platform,
|
|
system, install path and core."""
|
|
if not items:
|
|
return []
|
|
columns = (["Platform"] if platform_column else []) + [
|
|
"System", "Installed by", "Core", "Game", "Verdict", "Findings", "Record"]
|
|
return [
|
|
heading,
|
|
"",
|
|
intro.format(prefix=prefix, page=METHOD_PAGE),
|
|
"",
|
|
"| " + " | ".join(columns) + " |",
|
|
"|" + "|".join("---" for _ in columns) + "|",
|
|
*(_pack_row(item, platform_column) for item in
|
|
sorted(items, key=lambda i: (i["platform"], i["system"], i["install"], i["core"], i["content"]))),
|
|
"",
|
|
]
|
|
|
|
|
|
def pack_tested_section(items: list[dict]) -> list[str]:
|
|
"""The Tested block of a platform page."""
|
|
return packs_section(items, "## Tested", platform_column=False, prefix="../", intro=PLATFORM_INTRO)
|
|
|
|
|
|
# Characters a campaign value could use to leave its place on the page: HTML
|
|
# and MDX syntax ({ } opens an expression), code spans, emphasis and links.
|
|
_UNSAFE = {"&": "&", "<": "<", ">": ">", "{": "{", "}": "}", "`": "`",
|
|
"[": "[", "]": "]", "*": "*", "_": "_", "\\": "\", "|": "|"}
|
|
|
|
|
|
def _plain(value: object) -> str:
|
|
"""Text a run produced (core output, a romfetch error, a profile name),
|
|
shown as text: every character that is markup on the page is a
|
|
character reference."""
|
|
text = " ".join(str(value if value is not None else "").split())
|
|
return "".join(_UNSAFE.get(char, char) for char in text)
|
|
|
|
|
|
def _sentence(value: object) -> str:
|
|
"""A finding's statement as a sentence of the page: capital first."""
|
|
text = _plain(value)
|
|
return text[:1].upper() + text[1:]
|
|
|
|
|
|
def _code(value: object) -> str:
|
|
"""A value as code that cannot close its element or open an expression."""
|
|
return f"<code>{_plain(value)}</code>"
|
|
|
|
|
|
def _value(value: object) -> str:
|
|
return _code(json.dumps(value, ensure_ascii=False)) if value is not None else "nothing"
|
|
|
|
|
|
def campaign_section(summary: dict | None) -> list[str]:
|
|
"""The campaign's last pass and its open findings, for the explorer page."""
|
|
if not summary or not summary.get("last_run"):
|
|
return []
|
|
last = summary["last_run"]
|
|
counts = last["counts"]
|
|
lines = [
|
|
"## Campaign",
|
|
"",
|
|
f"Tests run by themselves every night on an idle computer ([how]({METHOD_PAGE})). "
|
|
f"Last pass: {_plain(last['ended_at'][:10])}, {_plain(last['result'])}"
|
|
+ (f" ({_plain(last['detail'])})" if last.get("detail") else "") + ". "
|
|
f"{int(counts['done'])} tests done, {int(counts['failed'])} failed, {int(counts['pending'])} waiting. "
|
|
f"{summary['applied']} profile fields were set from a run.",
|
|
"",
|
|
]
|
|
if last["pending_reasons"]:
|
|
lines += ["| Waiting for | Tests |", "|-------------|------:|"]
|
|
lines += [f"| {_plain(reason)} | {int(number)} |"
|
|
for reason, number in sorted(last["pending_reasons"].items(), key=lambda kv: (-kv[1], kv[0]))]
|
|
lines.append("")
|
|
if last["failed"]:
|
|
lines += ["Failed tests:", ""]
|
|
lines += [f"- {_plain(item['emulator'])}, {_code(item['check'])}: {_plain(item['reason'])}"
|
|
for item in last["failed"]]
|
|
lines.append("")
|
|
lines += ["### Open findings", ""]
|
|
if not summary["open"]:
|
|
lines += ["None: every finding was settled by a rule or resolved by a later run.", ""]
|
|
return lines
|
|
lines += ["A finding is what a run showed that no rule settles. It waits for someone to "
|
|
"read the emulator's code.", ""]
|
|
for item in summary["open"]:
|
|
records = ", ".join(f"[{_plain(rid)}]({link})" if link else _code(rid)
|
|
for rid, link in zip(item["evidence"], item["links"]))
|
|
lines += [
|
|
f"- **{_plain(item['emulator'])}**, {_code(item['file'])}, {_code(item['field'])} "
|
|
f"({_plain(item['status'])}): the profile says {_value(item['profile_says'])}, the run "
|
|
f"shows {_value(item['run_shows'])}. {_sentence(item['statement'])}. Records: {records}.",
|
|
"",
|
|
]
|
|
return lines
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# SQLite
|
|
# --------------------------------------------------------------------------
|
|
|
|
SCHEMA_SQL = """
|
|
PRAGMA journal_mode = OFF;
|
|
PRAGMA synchronous = OFF;
|
|
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
CREATE TABLE cores (
|
|
sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT,
|
|
build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT
|
|
);
|
|
CREATE TABLE hosts (
|
|
id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL,
|
|
kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL
|
|
);
|
|
CREATE TABLE emulator_systems (
|
|
emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system)
|
|
);
|
|
CREATE TABLE runs (
|
|
id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL,
|
|
kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT,
|
|
superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
|
|
host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
|
|
max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
|
|
script_modified INTEGER, flags TEXT NOT NULL, record_url TEXT NOT NULL,
|
|
line INTEGER NOT NULL,
|
|
migrated_from_version INTEGER
|
|
);
|
|
CREATE TABLE inputs (
|
|
run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT,
|
|
content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL,
|
|
warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL,
|
|
observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL
|
|
);
|
|
CREATE TABLE content_tracks (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
|
|
file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL,
|
|
PRIMARY KEY (run_id, position)
|
|
);
|
|
CREATE TABLE checks (
|
|
run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL,
|
|
observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL,
|
|
statement TEXT NOT NULL
|
|
);
|
|
CREATE TABLE check_groups (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
|
|
value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position)
|
|
);
|
|
CREATE TABLE variants (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
|
|
label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
|
|
flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
|
|
files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
|
|
);
|
|
CREATE TABLE system_files (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
|
|
destination TEXT NOT NULL, sha1 TEXT,
|
|
PRIMARY KEY (run_id, variant_position, destination)
|
|
);
|
|
CREATE TABLE benchmarks (
|
|
run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL,
|
|
median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL,
|
|
min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric)
|
|
);
|
|
CREATE INDEX runs_emulator_idx ON runs(emulator, subject);
|
|
CREATE INDEX runs_subject_idx ON runs(subject);
|
|
CREATE INDEX emulator_systems_system_idx ON emulator_systems(system);
|
|
CREATE INDEX checks_claim_idx ON checks(claim);
|
|
CREATE INDEX system_files_sha1_idx ON system_files(sha1);
|
|
"""
|
|
|
|
TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks",
|
|
"checks", "check_groups", "variants", "system_files", "benchmarks")
|
|
|
|
|
|
def _host_key(host: dict) -> tuple:
|
|
return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel",
|
|
"machine", "python"))
|
|
|
|
|
|
def write_sqlite(destination: Path, stored: list[mr.Stored], source: dict | None, profiles: dict,
|
|
scratch: Path) -> None:
|
|
"""A deterministic database of the records: sorted rows, no clock."""
|
|
scratch.mkdir(parents=True, exist_ok=True)
|
|
temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite"
|
|
temp.unlink(missing_ok=True)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
stored = sorted(stored, key=lambda s: (s.emulator, s.line))
|
|
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
|
|
hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)}
|
|
connection = sqlite3.connect(temp)
|
|
try:
|
|
connection.executescript(SCHEMA_SQL)
|
|
meta = {
|
|
"schema_version": SQLITE_SCHEMA_VERSION,
|
|
"record_version": str(mr.RECORD_VERSION),
|
|
"measurements_repo": (source or {}).get("repo") or "",
|
|
"measurements_commit": (source or {}).get("commit") or "",
|
|
"license": "MIT",
|
|
"scope": "core probe records, one per line of measurements/<emulator>.jsonl",
|
|
}
|
|
connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
|
|
cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
|
|
connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [
|
|
(sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")),
|
|
json_text(core.get("build_command")), json_text(core.get("compiler")),
|
|
json_text(core.get("library_name")), json_text(core.get("library_version")))
|
|
for sha, core in sorted(cores.items())])
|
|
connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [
|
|
(number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])])
|
|
emulators = sorted({s.emulator for s in stored})
|
|
connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [
|
|
(emu, system) for emu in emulators
|
|
for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))])
|
|
for entry in stored:
|
|
_insert_run(connection, entry, source or {}, hosts, entry.record["id"] not in live_ids)
|
|
connection.commit()
|
|
connection.execute("VACUUM")
|
|
finally:
|
|
connection.close()
|
|
os.replace(temp, destination)
|
|
|
|
|
|
def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
|
|
hosts: dict, superseded: bool) -> None:
|
|
r = entry.record
|
|
host, front, inputs = r["host"], r["frontend"], r["inputs"]
|
|
rid = r["id"]
|
|
connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
|
|
int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
|
|
json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
|
|
front.get("commit"), front["script_sha256"],
|
|
None if front.get("script_modified") is None else int(front["script_modified"]),
|
|
json_text(r["flags"]),
|
|
permalink(source.get("repo", ""), source.get("commit", ""), mr.records_path(entry.emulator), entry.line),
|
|
entry.line, r.get("migrated_from_version")))
|
|
content = inputs.get("content") or {}
|
|
connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, content.get("label"), content.get("sha1"), content.get("size"),
|
|
json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"],
|
|
inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]),
|
|
inputs["frame_hash_every"], inputs["joypad_ports"]))
|
|
connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [
|
|
(rid, position, track["file"], track["sha1"], track["size"])
|
|
for position, track in enumerate(content.get("tracks") or [])])
|
|
check = r.get("check")
|
|
if check:
|
|
result = check["result"]
|
|
connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, check["claim"], check["method"], check["observable"], check["variable"],
|
|
result["status"], result["statement"]))
|
|
connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
|
|
(rid, position, json_text(group["value"]),
|
|
mr.value_sha1(group["value"]) if isinstance(group["value"], (list, dict)) else None,
|
|
json_text(group["variants"]))
|
|
for position, group in enumerate(result["groups"])])
|
|
for position, variant in enumerate(r["variants"]):
|
|
reps = variant["repetitions"]
|
|
first = reps[0]
|
|
# The first repetition stands for the variant, its per-frame hashes left out.
|
|
observables = {key: value for key, value in (first.get("observables") or {}).items()
|
|
if key not in BULKY_OBSERVABLES}
|
|
connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)),
|
|
json_text(variant["flags"]), json_text(observables),
|
|
json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")),
|
|
json_text(first.get("tracing"))))
|
|
connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
|
|
(rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
|
|
for metric, stats in sorted((r.get("timing") or {}).items()):
|
|
connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", (
|
|
rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"]))
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Building the site's files
|
|
# --------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class Built:
|
|
index: dict
|
|
paths: list[Path]
|
|
|
|
|
|
def load_profiles(emulators_dir: Path) -> dict:
|
|
from common import load_emulator_profiles # noqa: PLC0415
|
|
|
|
return load_emulator_profiles(str(emulators_dir), skip_aliases=False)
|
|
|
|
|
|
def records_commit(root: Path) -> str | None:
|
|
"""The commit the record links name, with a warning when the records on
|
|
disk are not the ones it holds."""
|
|
commit = mr.last_commit(root)
|
|
if commit is None:
|
|
print(f"WARNING: no commit holds {root / mr.RECORDS_DIR}: the record links "
|
|
"lead nowhere until the records are committed.", file=sys.stderr)
|
|
return None
|
|
on_disk = {path: file.read_bytes() for path, file in mr.tree_files(root).items()}
|
|
if mr.committed_files(root, commit) != on_disk:
|
|
print(f"WARNING: {mr.RECORDS_DIR}/ differs from {commit[:12]}, the last commit that "
|
|
"touched it: links to records added since lead nowhere until they are "
|
|
"committed.", file=sys.stderr)
|
|
return commit
|
|
|
|
|
|
def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulators",
|
|
scratch: Path = ROOT / "tmp" / "site", system_names: dict[str, str] | None = None,
|
|
repo: str = REPO, commit: str | None = None) -> Built:
|
|
"""Write the index and the databases under the site's public folder.
|
|
|
|
``commit`` is the revision the record links name, by default the last
|
|
commit that touched the records.
|
|
"""
|
|
stored = mr.read_records(root)
|
|
source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)}
|
|
profiles = load_profiles(emulators_dir)
|
|
index = build_index(stored, source, profiles, system_names)
|
|
index["campaign"] = campaign_summary(root, stored, source)
|
|
index["packs"] = pack_summary(root, source)
|
|
paths = [public / INDEX_PATH]
|
|
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
|
|
write_if_changed(str(public / INDEX_PATH),
|
|
json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
|
|
write_sqlite(public / SQLITE_PATH, stored, source, profiles, scratch)
|
|
paths.append(public / SQLITE_PATH)
|
|
for emulator in sorted({s.emulator for s in stored}):
|
|
target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite"
|
|
write_sqlite(target, [s for s in stored if s.emulator == emulator], source, profiles, scratch)
|
|
paths.append(target)
|
|
return Built(index, paths)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Pages
|
|
# --------------------------------------------------------------------------
|
|
|
|
def by_emulator(index: dict) -> dict[str, list[dict]]:
|
|
out: dict[str, list[dict]] = {}
|
|
for item in index.get("items", []):
|
|
out.setdefault(item["emulator"], []).append(item)
|
|
return out
|
|
|
|
|
|
def _text(value: object) -> str:
|
|
"""Record text as the page shows it. A semicolon joining two clauses
|
|
ends a sentence on the site: the record keeps the words as written."""
|
|
text = " ".join(str(value or "").split())
|
|
text = re.sub(r";\s+(\w)", lambda m: ". " + m[1].upper(), text)
|
|
return _escape_tags(text)
|
|
|
|
|
|
def _short(commit: str | None) -> str:
|
|
return f"`{commit[:12]}`" if commit else "unknown"
|
|
|
|
|
|
def tested_item(item: dict) -> list[str]:
|
|
"""One test: what ran, the result, the method and host beside it, the record."""
|
|
title = item["claim"] or f"Benchmark of {item['content'] or 'the core alone'}"
|
|
runs = "1 run" if item["runs"] == 1 else f"{item['runs']} runs, the latest shown"
|
|
core = item["core"]
|
|
build = f"{_text(core['library'])} at {_short(core['commit'])}" if core.get("commit") \
|
|
else _text(core["library"])
|
|
lines = [
|
|
f"- **{_text(title)}**",
|
|
"",
|
|
f" **Result:** `{item['status']}`. {_text(item['statement'])}.",
|
|
"",
|
|
]
|
|
if item.get("method"):
|
|
lines += [f" **Method:** {_text(item['method'])}", ""]
|
|
lines += [
|
|
f" **Ran on:** {_text(item['host'])}, {item['date'][:10]}. Core build: {build}. "
|
|
f"{item['variants']} variant{'s' if item['variants'] != 1 else ''}, "
|
|
f"{item['repetitions']} repetition{'s' if item['repetitions'] != 1 else ''} each"
|
|
+ (f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" if item["flags"] else "")
|
|
+ ".",
|
|
"",
|
|
f" [Full record]({item['record_url']}), {runs}.",
|
|
"",
|
|
]
|
|
return lines
|
|
|
|
|
|
def tested_section(name: str, items: list[dict]) -> list[str]:
|
|
"""The Tested block of an emulator page, empty when nothing was recorded."""
|
|
if not items:
|
|
return []
|
|
lines = [
|
|
"## Tested",
|
|
"",
|
|
f"This emulator was run in real conditions by the [core probe](../{METHOD_PAGE}). "
|
|
f"Each test below is the latest of its kind. "
|
|
f"[All measurements](../{EXPLORER_PAGE}), "
|
|
f"[these records as SQLite](../downloads/measurements/{name}.sqlite).",
|
|
"",
|
|
]
|
|
for item in sorted(items, key=lambda i: (i["date"], i["subject"]), reverse=True):
|
|
lines += tested_item(item)
|
|
return lines
|
|
|
|
|
|
def hosts_table(index: dict) -> list[str]:
|
|
hosts: dict[str, int] = {}
|
|
for item in index.get("items", []):
|
|
# The load average differs per run: the computer is what comes before it.
|
|
computer = item["host"].split(", load ")[0]
|
|
hosts[computer] = hosts.get(computer, 0) + 1
|
|
if not hosts:
|
|
return ["No result is recorded yet.", ""]
|
|
return [
|
|
"| Computer | Tests |",
|
|
"|----------|------:|",
|
|
*(f"| {_text(host)} | {count} |" for host, count in sorted(hosts.items())),
|
|
"",
|
|
]
|
|
|
|
|
|
def with_hosts(page: str, index: dict) -> str:
|
|
"""The methodology page with the computers the results ran on."""
|
|
return page.replace(HOSTS_MARKER, "\n".join(hosts_table(index)).rstrip("\n"))
|
|
|
|
|
|
def generate_explorer_page(index: dict) -> str:
|
|
commit = index.get("commit")
|
|
repo = index.get("repo") or ""
|
|
source = (f"[`measurements/`]({repo}/tree/{commit}/{mr.RECORDS_DIR}) at {_short(commit)}"
|
|
if commit else f"`{mr.RECORDS_DIR}/` of this repository")
|
|
return "\n".join([
|
|
"# Measurements",
|
|
"",
|
|
f"Emulators run in real conditions: {index.get('records', 0)} records, "
|
|
f"{index.get('count', 0)} tests, read from {source}. "
|
|
f"[How they are made]({METHOD_PAGE}).",
|
|
"",
|
|
" ".join([
|
|
markup.button("SQLite of every record", "downloads/measurements.sqlite", primary=True),
|
|
markup.button("Index as JSON", "data/measurements/index.json"),
|
|
]),
|
|
"",
|
|
"Each row is the latest record of one test, with its method and the computer it "
|
|
"ran on. One SQLite file per emulator is linked from its page.",
|
|
"",
|
|
"<MeasurementsExplorer />",
|
|
"",
|
|
*packs_section(index.get("packs") or []),
|
|
*campaign_section(index.get("campaign")),
|
|
])
|
|
|
|
|
|
def sqlite_tables(path: Path) -> list[tuple[str, list[str]]]:
|
|
"""Each table of a database with its columns, in the order created."""
|
|
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
try:
|
|
names = [row[0] for row in connection.execute(
|
|
"SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")]
|
|
return [(name, [row[1] for row in connection.execute(f"PRAGMA table_info({name})")])
|
|
for name in names]
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def data_page_section(path: Path) -> list[str]:
|
|
"""The Data page's description of measurements.sqlite."""
|
|
return [
|
|
"## Measurements",
|
|
"",
|
|
"[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe "
|
|
f"record kept in [`{mr.RECORDS_DIR}/`]({REPO}/tree/main/{mr.RECORDS_DIR}), and "
|
|
"`downloads/measurements/<emulator>.sqlite` the records of one emulator, linked "
|
|
f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads "
|
|
"[`data/measurements/index.json`](data/measurements/index.json). A record line is "
|
|
"valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json).",
|
|
"",
|
|
"| Table | Columns |",
|
|
"|-------|---------|",
|
|
*(f"| `{name}` | {', '.join(f'`{c}`' for c in columns)} |" for name, columns in sqlite_tables(path)),
|
|
"",
|
|
"`meta` names the commit the record links point to and the record version. A list or object "
|
|
"value is JSON text, an absent one empty. `runs.superseded` is 1 for a record a "
|
|
"later one corrects.",
|
|
"",
|
|
"```sql",
|
|
"SELECT r.emulator, c.claim, c.status, r.recorded_at, h.cpu_model",
|
|
"FROM runs r JOIN checks c ON c.run_id = r.id JOIN hosts h ON h.id = r.host_id",
|
|
"WHERE r.superseded = 0 ORDER BY r.recorded_at;",
|
|
"```",
|
|
"",
|
|
]
|