kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-01 00:27:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: merge one-file measurement records

Browse Source
This commit is contained in:
Abdessamad Derraz committed 2026-10-27 10:24:41 +01:00
2 parents a9301fbdc0 da68f19411
commit 0fbf920e45
23 files changed
+972 -1483
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

scripts/check_freshness.py
+3 -31
View File
@@ -3,7 +3,7 @@
Each layer the repository transcribes moves on its own schedule: platform
lists, hardware targets, core-info, buildbot system assets, dump catalogs,
romset DATs, the measurements pin, the CI toolchain. This script asks each of them the same
romset DATs, the CI toolchain. This script asks each of them the same
question, is the local copy the one upstream serves, and answers with one
row per subject.
@@ -40,7 +40,7 @@ import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable, Iterator
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
@@ -80,8 +80,7 @@ INSTALLERS = ("install.sh", "install.ps1")
SHOWN_ITEMS = 4 # items named in a diff summary before ", ..."
SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..."
AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "measurements", "ci",
"profiles")
AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "ci", "profiles")
OK, STALE, UNKNOWN, ERROR, SKIPPED = "OK", "STALE", "UNKNOWN", "ERROR", "SKIPPED"
# Every status a finding may carry, in the order the summary counts them.
STATUSES = (STALE, UNKNOWN, ERROR, SKIPPED, OK)
@@ -915,31 +914,6 @@ def _check_buildbot_system() -> Finding:
detail=f"{names}{more} (_data_dirs.yml)", items=every)
# --- measurements -----------------------------------------------------------
def check_measurements(offline: bool, head: Callable[[str], str] | None = None,
pin_path: Path | None = None) -> Finding:
"""The measurements pin against the head of the measurements repository.
A record added there reaches the site only once the pin moves to it.
"""
import sync_measurements # noqa: PLC0415
pin = sync_measurements.load_pin(pin_path or sync_measurements.PIN_FILE)
local = pin["commit"][:12]
if offline:
return Finding("measurements", "records pin", SKIPPED, local=local, detail="offline")
try:
remote = (head or sync_measurements.remote_head)(pin["repo"])
except (sync_measurements.MeasurementsError, OSError, subprocess.TimeoutExpired) as exc:
return Finding("measurements", "records pin", UNKNOWN, local=local, detail=str(exc))
if remote == pin["commit"]:
return Finding("measurements", "records pin", OK, local=local, remote=remote[:12])
return Finding("measurements", "records pin", STALE, local=local, remote=remote[:12],
detail="newer records: python scripts/sync_measurements.py --fetch --bump")
# --- catalogs and recipes ---------------------------------------------------
@@ -1532,8 +1506,6 @@ def main() -> int:
findings.extend(check_data(args.offline))
if "catalogs" in areas:
findings.extend(check_catalogs(args.cache_dir, args.offline))
if "measurements" in areas:
findings.append(check_measurements(args.offline))
if "ci" in areas:
findings.extend(check_ci(args.cache_dir, args.offline))
if "profiles" in areas:
scripts/core_probe.py
+13 -48
View File
@@ -10,10 +10,9 @@ Three modes share one runner:
Each repetition runs in its own process, under strace when it is installed,
so the files a core opened are read from the kernel rather than guessed.
`--record DIR` appends one record per invocation to a clone of the
measurements repository, carrying what is needed to reproduce it (core build,
frontend commit, host, inputs by hash). measurement_records.py splits it into
a slim line and a gzip sidecar.
`--record` appends one record per invocation, as one line of
measurements/<emulator>.jsonl in this repository, carrying what is needed to
reproduce it (core build, frontend commit, host, inputs by hash).
"""
from __future__ import annotations
@@ -39,7 +38,6 @@ from dataclasses import dataclass, field
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
MEASUREMENTS_CLONE = ROOT / ".cache" / "measurements"
MIN_BENCH_REPETITIONS = 5
DEFAULT_WORK_DIR = ROOT / "tmp" / "core_probe"
BASE_OBSERVABLES = ("frame_sha1", "video_sha1", "audio_sha1", "state_sha1")
@@ -771,10 +769,9 @@ def build_parser() -> argparse.ArgumentParser:
common.add_argument("--core-repo", help="source repository URL of the core build")
common.add_argument("--core-commit", help="commit the core was built from")
common.add_argument("--build-command", help="command and flags that built the core")
common.add_argument("--record", metavar="DIR",
help="append the record to the measurements clone DIR "
f"(records/<emulator>.jsonl and data/<emulator>/), "
f"for example {MEASUREMENTS_CLONE.relative_to(ROOT)}")
common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
help="append the record as one line of measurements/<emulator>.jsonl "
"in the checkout DIR (default: this one)")
common.add_argument("--emulator", help="profile the record is filed under (default: the "
"profile whose cores: names the .so, else the "
"core's library name)")
@@ -823,7 +820,6 @@ def validate_args(args: argparse.Namespace) -> list[str]:
if args.png_dir and not args.frame_hash_every:
problems.append("--png-dir needs --frame-hash-every")
if args.record:
problems.extend(record_target_problems(Path(args.record)))
if not args.emulator and Path(args.core).is_file():
try:
default_emulator(Path(args.core), None)
@@ -907,32 +903,6 @@ def check_result(variants: list[dict], observable: str) -> dict:
}
def record_target_problems(directory: Path) -> list[str]:
"""Why records may not be written under ``directory``.
Records live in their own repository. A directory inside this one's
tracked tree would put them in its history, megabytes per campaign;
an ignored directory (tmp/, the clone under .cache/) is fine.
"""
target = directory.resolve()
if target != ROOT and ROOT not in target.parents:
return []
try:
ignored = subprocess.run(["git", "check-ignore", "-q", str(target)], cwd=ROOT,
capture_output=True, check=False).returncode
except OSError as exc:
return [f"--record {directory}: cannot ask git whether it is tracked ({exc})"]
if ignored == 0:
return []
if ignored != 1:
# 128: not a git checkout, nothing to keep records out of.
return []
return [f"--record {directory}: inside the tracked tree of this repository. Records go "
f"to the measurements repository: clone it with "
f"`python scripts/sync_measurements.py --fetch` and pass "
f"--record {MEASUREMENTS_CLONE.relative_to(ROOT)}"]
def default_emulator(core_file: Path, library_name: str | None) -> str:
"""The profile whose cores: names the .so, else the library name as a slug."""
stem = core_file.name.split(".")[0].removesuffix("_libretro")
@@ -966,24 +936,19 @@ def default_subject(record: dict) -> str:
return f"{record['kind']}-{slugify(content, 30)}-{digest}"
def write_record(directory: Path, full: dict) -> tuple[Path, Path]:
"""Append the record to its emulator's .jsonl and write its sidecar."""
from measurement_records import ( # noqa: PLC0415
line_text, parse_lines, records_path, seal, sidecar_path, split)
def write_record(directory: Path, full: dict) -> Path:
"""Append the record, with its id, to its emulator's .jsonl."""
from measurement_records import line_text, parse_lines, records_path, seal # noqa: PLC0415
slim, sidecar = split(full)
sealed, blob = seal(slim, sidecar)
sealed = seal(full)
lines = directory / records_path(sealed["emulator"])
existing = parse_lines(lines.read_text(encoding="utf-8"), str(lines)) if lines.is_file() else []
if sealed["supersedes"] and sealed["supersedes"] not in {r["id"] for r in existing}:
raise ProbeError(f"--supersedes {sealed['supersedes']}: no such record in {lines}")
data = directory / sidecar_path(sealed["emulator"], sealed["id"])
data.parent.mkdir(parents=True, exist_ok=True)
data.write_bytes(blob)
lines.parent.mkdir(parents=True, exist_ok=True)
with lines.open("a", encoding="utf-8") as handle:
handle.write(line_text(sealed) + "\n")
return lines, data
return lines
def probe(args: argparse.Namespace) -> int:
@@ -1060,11 +1025,11 @@ def probe(args: argparse.Namespace) -> int:
record["subject"] = args.subject or default_subject(record)
if args.record:
try:
lines, data = write_record(Path(args.record), record)
lines = write_record(Path(args.record), record)
except ProbeError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 2
print(f"recorded {lines} (line {sum(1 for _ in lines.open(encoding='utf-8'))}) and {data}",
print(f"recorded {lines} (line {sum(1 for _ in lines.open(encoding='utf-8'))})",
file=sys.stderr)
print(json.dumps(record, indent=2, sort_keys=False))
return 0
scripts/generate_site.py
+4 -10
View File
@@ -110,7 +110,6 @@ from siteguides import (
)
import release_record
import sitemeasurements
import sync_measurements
# The Starlight project, and where its pages go inside it.
STARLIGHT_DIR = "website"
@@ -11007,11 +11006,6 @@ def main() -> None:
help="directory the pages are written to (default: the project's "
"content collection, src/content/docs)",
)
parser.add_argument(
"--require-measurements", action="store_true",
help="fail when the measurements clone does not hold the pinned commit "
"(the deploy build passes it; without it the site lists no measurements)",
)
args = parser.parse_args()
if args.docs_dir is None:
args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT)
@@ -11168,9 +11162,9 @@ def _generate(args: argparse.Namespace) -> None:
if clash:
raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}")
print("Compiling the measurements at their pin...")
measured = sync_measurements.build(
public, emulators_dir=Path(args.emulators_dir), require=args.require_measurements,
print("Compiling the measurements...")
measured = sitemeasurements.build(
public, emulators_dir=Path(args.emulators_dir), repo=REPO_URL,
system_names={sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))},
)
for path in measured.paths:
@@ -11187,7 +11181,7 @@ def _generate(args: argparse.Namespace) -> None:
)
write_release_advertised(Path(args.site_dir) / "src" / "generated")
write_if_changed(str(docs / "data.md"),
generate_data_page(exports, public / sync_measurements.SQLITE_PATH)
generate_data_page(exports, public / sitemeasurements.SQLITE_PATH)
+ reference_next())
PAGE_DATA["data.md"] = dataset_structured_data(exports, db)
scripts/measurement_records.py
+149 -150
View File
@@ -1,50 +1,40 @@
"""Core probe records as the measurements repository stores them.
"""Core probe records as this repository stores them.
A run of core_probe.py produces one full record. It is stored as two parts:
A run of core_probe.py produces one full record, stored as one line:
records/<emulator>.jsonl one slim record per line, append-only
data/<emulator>/<id>.json.gz the bulky rest, named by the record id
measurements/<emulator>.jsonl one record per line, append-only
The slim record keeps what a reader needs to judge and reproduce the result:
claim, method, observable, result, statistics, host, inputs by hash, core
build and frontend commit, and per variant the values of its first
repetition. The sidecar holds every repetition in full (per-frame hashes,
core logs, declared options), the raw timing values the statistics were
computed from, and any compared value too large to sit in a line. Its
sha256 is in the record, so the line vouches for it. ``join`` puts the two
back into the full record, ``split`` is its inverse.
A line holds the whole run: claim, method, observable, result, host, inputs
by hash, core build and frontend commit, and every repetition of every
variant in full (per-frame hashes, core logs, declared options), with the
raw timing values the statistics were computed from.
A record id is the date and the first 12 hex digits of the sha256 of the
record without its id. A record carried over from the first layout keeps
the id it had, which hashes that full record.
record without its id, in canonical JSON. A record carried over from the
first layout keeps the id it had, which hashes that full record.
Standard library only: core_probe.py, sync_measurements.py and the site
Standard library only: core_probe.py, validate_schemas.py and the site
read it.
"""
from __future__ import annotations
import copy
import gzip
import hashlib
import json
import re
import subprocess
from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
RECORD_VERSION = 2
MEASUREMENTS_REPO = "https://github.com/Abdess/retrobios-measurements"
RECORDS_DIR = "records"
DATA_DIR = "data"
# Fields a repetition keeps in the slim record, per variant, from the first one.
VARIANT_SUMMARY = ("observables", "files_opened", "files_probed_absent", "tracing")
# Observables too large for a line: their values stay in the sidecar.
BULKY_OBSERVABLES = ("frame_hashes",)
RECORD_VERSION = 3
RECORDS_DIR = "measurements"
ID_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$")
SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:[-_][a-z0-9]+)*$")
class RecordError(ValueError):
"""A record or sidecar that does not match its counterpart."""
"""A record line that cannot be read."""
def canonical(value: object) -> bytes:
@@ -55,6 +45,11 @@ def sha256_hex(data: bytes) -> str:
return hashlib.sha256(data).hexdigest()
def value_sha1(value: object) -> str:
"""The sha1 of a value in canonical JSON."""
return hashlib.sha1(canonical(value)).hexdigest()
def slugify(text: str, limit: int = 40) -> str:
"""Lowercase words joined by hyphens, cut at a word boundary."""
words = re.findall(r"[a-z0-9]+", text.lower())
@@ -67,151 +62,55 @@ def slugify(text: str, limit: int = 40) -> str:
return out or "record"
def record_id(slim: dict) -> str:
body = {key: value for key, value in slim.items() if key != "id"}
return f"{slim['recorded_at'][:10]}-{sha256_hex(canonical(body))[:12]}"
def record_id(record: dict) -> str:
body = {key: value for key, value in record.items() if key != "id"}
return f"{record['recorded_at'][:10]}-{sha256_hex(canonical(body))[:12]}"
def line_text(slim: dict) -> str:
def seal(record: dict) -> dict:
"""The record with its id."""
return {**record, "id": record_id(record)}
def line_text(record: dict) -> str:
"""The record as one line of its .jsonl file, keys in a fixed order."""
return json.dumps(slim, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
def sidecar_path(emulator: str, rid: str) -> str:
return f"{DATA_DIR}/{emulator}/{rid}.json.gz"
return json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
def records_path(emulator: str) -> str:
return f"{RECORDS_DIR}/{emulator}.jsonl"
def sidecar_bytes(sidecar: dict) -> bytes:
"""gzip of the sidecar JSON, with no name and no time in the header."""
return gzip.compress(canonical(sidecar), compresslevel=9, mtime=0)
IDENTITY = ("id", "emulator", "subject", "supersedes", "migrated_from_version")
def read_sidecar(blob: bytes) -> dict:
return json.loads(gzip.decompress(blob))
def _value_digest(value: object) -> str:
return hashlib.sha1(canonical(value)).hexdigest()
def split(full: dict) -> tuple[dict, dict]:
"""The full record as (slim record without id and data, sidecar)."""
slim = copy.deepcopy(full)
sidecar: dict = {"variants": [], "timing_raw": {}, "group_values": {}}
variants = []
for variant in slim["variants"]:
reps = variant["repetitions"]
sidecar["variants"].append({"label": variant["label"], "repetitions": reps})
first = reps[0]
summary = {key: first.get(key) for key in VARIANT_SUMMARY}
summary["observables"] = {
key: value for key, value in (first.get("observables") or {}).items()
if key not in BULKY_OBSERVABLES
}
variants.append({
"label": variant["label"],
"system_files": variant["system_files"],
"flags": variant["flags"],
"repetitions": len(reps),
"loaded": all(rep.get("loaded") for rep in reps),
**summary,
})
slim["variants"] = variants
for name, stats in (slim.get("timing") or {}).items():
sidecar["timing_raw"][name] = stats.pop("raw")
for group in ((slim.get("check") or {}).get("result") or {}).get("groups", []):
if isinstance(group.get("value"), (list, dict)):
digest = _value_digest(group["value"])
sidecar["group_values"][digest] = group.pop("value")
group["value_sha1"] = digest
return slim, sidecar
def join(slim: dict, sidecar: dict) -> dict:
"""The full record a slim record and its sidecar were split from."""
full = copy.deepcopy(slim)
full.pop("id", None)
full.pop("data", None)
labels = [v["label"] for v in full["variants"]]
if labels != [v["label"] for v in sidecar["variants"]]:
raise RecordError(f"variants differ: record {labels}, sidecar "
f"{[v['label'] for v in sidecar['variants']]}")
full["variants"] = [
{"label": variant["label"], "system_files": variant["system_files"],
"repetitions": copy.deepcopy(stored["repetitions"]), "flags": variant["flags"]}
for variant, stored in zip(full["variants"], sidecar["variants"])
]
for name, raw in sidecar.get("timing_raw", {}).items():
full["timing"][name]["raw"] = list(raw)
for group in ((full.get("check") or {}).get("result") or {}).get("groups", []):
digest = group.pop("value_sha1", None)
if digest is not None:
group["value"] = copy.deepcopy(sidecar["group_values"][digest])
return full
def seal(slim: dict, sidecar: dict, rid: str | None = None) -> tuple[dict, bytes]:
"""The slim record with its data reference and id, and the sidecar bytes.
``rid`` keeps an id given before (a record carried over from the first
layout), otherwise the id is computed.
"""
blob = sidecar_bytes(sidecar)
sealed = {**slim, "data": {"sha256": sha256_hex(blob), "bytes": len(blob)}}
sealed["id"] = rid or record_id(sealed)
return sealed, blob
def check_pair(slim: dict, blob: bytes) -> list[str]:
"""What is wrong between a stored line and its stored sidecar."""
problems = []
data = slim.get("data") or {}
if sha256_hex(blob) != data.get("sha256"):
problems.append(f"{slim.get('id')}: sidecar sha256 {sha256_hex(blob)} != {data.get('sha256')}")
if len(blob) != data.get("bytes"):
problems.append(f"{slim.get('id')}: sidecar is {len(blob)} bytes, record says {data.get('bytes')}")
if not problems:
try:
full = join(slim, read_sidecar(blob))
except (RecordError, KeyError, OSError, ValueError) as exc:
return [f"{slim.get('id')}: record and sidecar do not join: {exc}"]
if slim.get("migrated_from_version") == 1:
digest = sha256_hex(canonical(first_layout(full)))[:12]
if str(slim.get("id", "")).rsplit("-", 1)[-1] != digest:
problems.append(f"{slim.get('id')}: id does not hash the first-layout record ({digest})")
return problems
V2_IDENTITY = ("emulator", "subject", "supersedes", "migrated_from_version")
def first_layout(full: dict) -> dict:
"""A full record as the first layout wrote it: no emulator, subject or
def first_layout(record: dict) -> dict:
"""A record as the first layout wrote it: no id, emulator, subject or
supersedes, record_version 1."""
old = {key: value for key, value in full.items() if key not in V2_IDENTITY}
old = {key: value for key, value in record.items() if key not in IDENTITY}
old["record_version"] = 1
return old
def check_line(slim: dict) -> list[str]:
def check_line(record: dict) -> list[str]:
"""The structural rules a line holds, without a JSON Schema validator."""
rid = slim.get("id", "")
rid = record.get("id", "")
problems = []
if slim.get("record_version") != RECORD_VERSION:
problems.append(f"{rid}: record_version {slim.get('record_version')} != {RECORD_VERSION}")
if record.get("record_version") != RECORD_VERSION:
problems.append(f"{rid}: record_version {record.get('record_version')} != {RECORD_VERSION}")
if not ID_PATTERN.match(str(rid)):
problems.append(f"{rid!r}: not an id of the form YYYY-MM-DD-<12 hex>")
elif str(slim.get("recorded_at", ""))[:10] != rid[:10]:
problems.append(f"{rid}: id date differs from recorded_at {slim.get('recorded_at')}")
elif slim.get("migrated_from_version") is None and record_id(slim) != rid:
problems.append(f"{rid}: id does not hash the record ({record_id(slim)})")
elif str(record.get("recorded_at", ""))[:10] != rid[:10]:
problems.append(f"{rid}: id date differs from recorded_at {record.get('recorded_at')}")
elif record.get("migrated_from_version") == 1:
digest = sha256_hex(canonical(first_layout(record)))[:12]
if not rid.endswith(f"-{digest}"):
problems.append(f"{rid}: id does not hash the first-layout record ({digest})")
elif record_id(record) != rid:
problems.append(f"{rid}: id does not hash the record ({record_id(record)})")
for key in ("emulator", "subject"):
if not SLUG_PATTERN.match(str(slim.get(key, ""))):
problems.append(f"{rid}: {key} {slim.get(key)!r} is not a lowercase slug")
if not SLUG_PATTERN.match(str(record.get(key, ""))):
problems.append(f"{rid}: {key} {record.get(key)!r} is not a lowercase slug")
return problems
@@ -244,3 +143,103 @@ def live(records: Iterable[dict]) -> list[dict]:
records = list(records)
superseded = {r["supersedes"] for r in records if r.get("supersedes")}
return [r for r in records if r["id"] not in superseded]
class MeasurementsError(RuntimeError):
"""The records of a tree cannot be read, or break a rule."""
@dataclass(frozen=True)
class Stored:
"""One record line as read from the tree."""
emulator: str
line: int
record: dict
def tree_files(root: Path) -> dict[str, Path]:
"""Repository path -> file of everything under ``root``/measurements."""
base = root / RECORDS_DIR
if not base.is_dir():
return {}
return {path.relative_to(root).as_posix(): path
for path in sorted(base.rglob("*")) if path.is_file()}
def read_records(root: Path) -> list[Stored]:
"""Every record line under ``root``, each checked against its rules."""
files = tree_files(root)
problems: list[str] = []
stored: list[Stored] = []
for path in sorted(files):
emulator = path[len(RECORDS_DIR) + 1:].removesuffix(".jsonl")
if not path.endswith(".jsonl") or "/" in emulator:
problems.append(f"{path}: {RECORDS_DIR}/ holds <emulator>.jsonl files only")
continue
seen: set[str] = set()
try:
records = parse_lines(files[path].read_text(encoding="utf-8"), path)
except RecordError as exc:
problems.append(str(exc))
continue
for number, record in enumerate(records, 1):
where = f"{path}:{number}"
problems.extend(f"{where}: {p}" for p in check_line(record))
if record.get("emulator") != emulator:
problems.append(f"{where}: emulator {record.get('emulator')!r} in {path}")
if record.get("id") in seen:
problems.append(f"{where}: id {record.get('id')} appears twice")
if record.get("supersedes") and record["supersedes"] not in seen:
problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file")
seen.add(record.get("id"))
stored.append(Stored(emulator, number, record))
if problems:
raise MeasurementsError("records under {}:\n {}".format(root, "\n ".join(problems)))
return stored
def _git(root: Path, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
def committed_files(root: Path, ref: str) -> dict[str, bytes]:
"""Path -> content of every file under measurements/ at ``ref``."""
listing = _git(root, "ls-tree", "-r", "-z", "--name-only", ref, "--", RECORDS_DIR)
if listing.returncode != 0:
raise MeasurementsError(f"git ls-tree {ref}: {listing.stderr.decode(errors='replace').strip()}")
out = {}
for path in filter(None, listing.stdout.decode("utf-8").split("\0")):
blob = _git(root, "show", f"{ref}:{path}")
if blob.returncode != 0:
raise MeasurementsError(f"git show {ref}:{path}: {blob.stderr.decode(errors='replace').strip()}")
out[path] = blob.stdout
return out
def append_only_problems(root: Path, ref: str = "HEAD") -> list[str]:
"""What the tree under ``root`` changed or removed of the lines ``ref``
holds. Lines added at the end and new files are allowed."""
before, after = committed_files(root, ref), tree_files(root)
problems = []
for path, content in sorted(before.items()):
if path not in after:
problems.append(f"{path}: removed")
continue
now = after[path].read_bytes()
if now == content:
continue
if path.endswith(".jsonl"):
moved = appended_only(content.decode("utf-8"), now.decode("utf-8"))
if moved:
problems.append(f"{path}: {moved}")
else:
problems.append(f"{path}: rewritten")
return problems
def last_commit(root: Path) -> str | None:
"""The last commit that touched the records: the one their permalinks name."""
proc = _git(root, "log", "-1", "--format=%H", "--", RECORDS_DIR)
commit = proc.stdout.decode().strip() if proc.returncode == 0 else ""
return commit if re.fullmatch(r"[0-9a-f]{40}", commit) else None
scripts/refresh_stale.py
+1 -9
View File
@@ -27,9 +27,6 @@ Mapping (stale -> command):
--source mame --fetch
catalogs/fbneo recipes python -m scripts.scraper.romset_dat_importer
--source fbneo --fetch
measurements/records pin
python scripts/sync_measurements.py --fetch --bump,
which refuses a remote that rewrote a pinned line
A platform that inherits from a rescraped one (retropie and lakka from
retroarch) reads the parent's file: its jobs start once the parent's chain
@@ -66,7 +63,7 @@ CHECK_FRESHNESS = REPO_ROOT / "scripts" / "check_freshness.py"
PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs", "measurements")
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
MANUAL_AREAS = ("coreinfo", "ci", "profiles")
JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10
@@ -410,11 +407,6 @@ def _command_for(
# redump / no-intro / tosec need manual DAT download.
return None
if area == "measurements":
# --fetch refuses a remote that rewrote a pinned line, and --bump
# only rewrites measurements-pin.json: nothing is committed.
return [sys.executable, "scripts/sync_measurements.py", "--fetch", "--bump"]
# coreinfo, ci, profiles: never auto-refresh.
return None
scripts/sitemeasurements.py
+369 -13
View File
@@ -1,24 +1,382 @@
"""The site's measurement pages, from the index sync_measurements.py writes.
"""The measurements the site shows, from the records under measurements/.
Three places read the same index: the Tested block of each emulator page,
the Measurements explorer page and the host table of the methodology page.
A number is never shown without its method and host beside it.
The records are core probe runs, one append-only .jsonl per emulator with one
full record per line (measurement_records.py). From them the build writes,
never committed:
public/data/measurements/index.json the compact index the explorer and
the emulator pages read, one item
per emulator and subject
public/downloads/measurements.sqlite every record, normalised
public/downloads/measurements/<emu>.sqlite one database per emulator
Three places read the index: the Tested block of each emulator page, the
Measurements explorer page and the host table of the methodology page. A
number is never shown without its method and host beside it. A record links
its line at the last commit that touched measurements/.
"""
from __future__ import annotations
import json
import os
import re
import sqlite3
import sys
from dataclasses import dataclass
from pathlib import Path
import measurement_records as mr
import sitemarkup as markup
from artifacts import json_text, write_if_changed
from siterender import _escape_tags
ROOT = Path(__file__).resolve().parent.parent
REPO = "https://github.com/Abdess/retrobios"
EXPLORER_PAGE = "measurements.md"
METHOD_PAGE = "wiki/measurements.md"
HOSTS_MARKER = "<!-- measurements-hosts -->"
INDEX_PATH = Path("data") / "measurements" / "index.json"
SQLITE_PATH = Path("downloads") / "measurements.sqlite"
PER_EMULATOR_DIR = Path("downloads") / "measurements"
SQLITE_SCHEMA_VERSION = "2"
# Observables the variants table leaves out: one value per sampled frame.
BULKY_OBSERVABLES = ("frame_hashes",)
INDEX_SCHEMA_VERSION = 1
# --------------------------------------------------------------------------
# Index
# --------------------------------------------------------------------------
def host_summary(host: dict) -> str:
parts = [host.get("cpu_model") or "unknown CPU"]
if host.get("cpu_count"):
parts.append(f"{host['cpu_count']} CPUs")
parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip()
if "Linux" in host.get("os", "") else host.get("os", ""))
parts.append(f"Python {host.get('python', '')}")
before, after = host.get("load_before"), host.get("load_after")
if before and after:
parts.append(f"load {before[0]} before, {after[0]} after")
return ", ".join(part for part in parts if part)
def bench_statement(record: dict) -> str:
timing = record["timing"]
frame, fps = timing["ms_per_frame"], timing["fps"]
return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), "
f"min {frame['min']:.3f}, max {frame['max']:.3f}, over "
f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames")
def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str:
return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
system_names: dict[str, str]) -> dict:
record = entry.record
profile = profiles.get(entry.emulator) or {}
check = record.get("check") or {}
core = record["core"]
content = (record["inputs"].get("content") or {})
return {
"emulator": entry.emulator,
"emulator_name": str(profile.get("emulator", entry.emulator)),
"profiled": bool(profile),
"systems": sorted(set(profile.get("systems") or [])),
"system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))],
"subject": record["subject"],
"kind": record["kind"],
"claim": check.get("claim"),
"method": check.get("method"),
"observable": check.get("observable"),
"variable": check.get("variable"),
"status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
"statement": (check.get("result") or {}).get("statement")
or (bench_statement(record) if record.get("timing") else ""),
"content": content.get("label"),
"variants": len(record["variants"]),
"repetitions": record["inputs"]["repetitions"],
"flags": len(record["flags"]),
"date": record["recorded_at"],
"host": host_summary(record["host"]),
"core": {
"library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])),
"file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"),
"commit": core.get("commit"), "build": core.get("build_command"),
},
"frontend_commit": record["frontend"].get("commit"),
"id": record["id"],
"runs": runs,
"record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line),
"sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
}
def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
system_names: dict[str, str] | None = None) -> dict:
"""One item per (emulator, subject): its latest record not superseded."""
by_subject: dict[tuple[str, str], list[mr.Stored]] = {}
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
for entry in stored:
if entry.record["id"] in live_ids:
by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry)
items = []
for key in sorted(by_subject):
entries = by_subject[key]
latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
items.append(index_item(latest, len(entries), source or {}, profiles, system_names or {}))
items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
return {
"schema_version": INDEX_SCHEMA_VERSION,
"kind": "measurements",
"repo": (source or {}).get("repo"),
"commit": (source or {}).get("commit"),
"records": len(stored),
"count": len(items),
"items": items,
}
# --------------------------------------------------------------------------
# SQLite
# --------------------------------------------------------------------------
SCHEMA_SQL = """
PRAGMA journal_mode = OFF;
PRAGMA synchronous = OFF;
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE cores (
sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT,
build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT
);
CREATE TABLE hosts (
id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL,
kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL
);
CREATE TABLE emulator_systems (
emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system)
);
CREATE TABLE runs (
id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL,
kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT,
superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
script_modified INTEGER, flags TEXT NOT NULL, record_url TEXT NOT NULL,
line INTEGER NOT NULL,
migrated_from_version INTEGER
);
CREATE TABLE inputs (
run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT,
content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL,
warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL,
observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL
);
CREATE TABLE content_tracks (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL,
PRIMARY KEY (run_id, position)
);
CREATE TABLE checks (
run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL,
observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL,
statement TEXT NOT NULL
);
CREATE TABLE check_groups (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position)
);
CREATE TABLE variants (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
);
CREATE TABLE system_files (
run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
destination TEXT NOT NULL, sha1 TEXT,
PRIMARY KEY (run_id, variant_position, destination)
);
CREATE TABLE benchmarks (
run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL,
median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL,
min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric)
);
CREATE INDEX runs_emulator_idx ON runs(emulator, subject);
CREATE INDEX runs_subject_idx ON runs(subject);
CREATE INDEX emulator_systems_system_idx ON emulator_systems(system);
CREATE INDEX checks_claim_idx ON checks(claim);
CREATE INDEX system_files_sha1_idx ON system_files(sha1);
"""
TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks",
"checks", "check_groups", "variants", "system_files", "benchmarks")
def _host_key(host: dict) -> tuple:
return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel",
"machine", "python"))
def write_sqlite(destination: Path, stored: list[mr.Stored], source: dict | None, profiles: dict,
scratch: Path) -> None:
"""A deterministic database of the records: sorted rows, no clock."""
scratch.mkdir(parents=True, exist_ok=True)
temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite"
temp.unlink(missing_ok=True)
destination.parent.mkdir(parents=True, exist_ok=True)
stored = sorted(stored, key=lambda s: (s.emulator, s.line))
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)}
connection = sqlite3.connect(temp)
try:
connection.executescript(SCHEMA_SQL)
meta = {
"schema_version": SQLITE_SCHEMA_VERSION,
"record_version": str(mr.RECORD_VERSION),
"measurements_repo": (source or {}).get("repo") or "",
"measurements_commit": (source or {}).get("commit") or "",
"license": "MIT",
"scope": "core probe records, one per line of measurements/<emulator>.jsonl",
}
connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [
(sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")),
json_text(core.get("build_command")), json_text(core.get("compiler")),
json_text(core.get("library_name")), json_text(core.get("library_version")))
for sha, core in sorted(cores.items())])
connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [
(number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])])
emulators = sorted({s.emulator for s in stored})
connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [
(emu, system) for emu in emulators
for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))])
for entry in stored:
_insert_run(connection, entry, source or {}, hosts, entry.record["id"] not in live_ids)
connection.commit()
connection.execute("VACUUM")
finally:
connection.close()
os.replace(temp, destination)
def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
hosts: dict, superseded: bool) -> None:
r = entry.record
host, front, inputs = r["host"], r["frontend"], r["inputs"]
rid = r["id"]
connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
front.get("commit"), front["script_sha256"],
None if front.get("script_modified") is None else int(front["script_modified"]),
json_text(r["flags"]),
permalink(source.get("repo", ""), source.get("commit", ""), mr.records_path(entry.emulator), entry.line),
entry.line, r.get("migrated_from_version")))
content = inputs.get("content") or {}
connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, content.get("label"), content.get("sha1"), content.get("size"),
json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"],
inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]),
inputs["frame_hash_every"], inputs["joypad_ports"]))
connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [
(rid, position, track["file"], track["sha1"], track["size"])
for position, track in enumerate(content.get("tracks") or [])])
check = r.get("check")
if check:
result = check["result"]
connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, check["claim"], check["method"], check["observable"], check["variable"],
result["status"], result["statement"]))
connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
(rid, position, json_text(group["value"]),
mr.value_sha1(group["value"]) if isinstance(group["value"], (list, dict)) else None,
json_text(group["variants"]))
for position, group in enumerate(result["groups"])])
for position, variant in enumerate(r["variants"]):
reps = variant["repetitions"]
first = reps[0]
# The first repetition stands for the variant, its per-frame hashes left out.
observables = {key: value for key, value in (first.get("observables") or {}).items()
if key not in BULKY_OBSERVABLES}
connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)),
json_text(variant["flags"]), json_text(observables),
json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")),
json_text(first.get("tracing"))))
connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
(rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
for metric, stats in sorted((r.get("timing") or {}).items()):
connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"]))
# --------------------------------------------------------------------------
# Building the site's files
# --------------------------------------------------------------------------
@dataclass
class Built:
index: dict
paths: list[Path]
def load_profiles(emulators_dir: Path) -> dict:
from common import load_emulator_profiles # noqa: PLC0415
return load_emulator_profiles(str(emulators_dir), skip_aliases=False)
def records_commit(root: Path) -> str | None:
"""The commit the record links name, with a warning when the records on
disk are not the ones it holds."""
commit = mr.last_commit(root)
if commit is None:
print(f"WARNING: no commit holds {root / mr.RECORDS_DIR}: the record links "
"lead nowhere until the records are committed.", file=sys.stderr)
return None
on_disk = {path: file.read_bytes() for path, file in mr.tree_files(root).items()}
if mr.committed_files(root, commit) != on_disk:
print(f"WARNING: {mr.RECORDS_DIR}/ differs from {commit[:12]}, the last commit that "
"touched it: links to records added since lead nowhere until they are "
"committed.", file=sys.stderr)
return commit
def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulators",
scratch: Path = ROOT / "tmp" / "site", system_names: dict[str, str] | None = None,
repo: str = REPO, commit: str | None = None) -> Built:
"""Write the index and the databases under the site's public folder.
``commit`` is the revision the record links name, by default the last
commit that touched the records.
"""
stored = mr.read_records(root)
source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)}
profiles = load_profiles(emulators_dir)
index = build_index(stored, source, profiles, system_names)
paths = [public / INDEX_PATH]
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
write_if_changed(str(public / INDEX_PATH),
json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
write_sqlite(public / SQLITE_PATH, stored, source, profiles, scratch)
paths.append(public / SQLITE_PATH)
for emulator in sorted({s.emulator for s in stored}):
target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite"
write_sqlite(target, [s for s in stored if s.emulator == emulator], source, profiles, scratch)
paths.append(target)
return Built(index, paths)
# --------------------------------------------------------------------------
# Pages
# --------------------------------------------------------------------------
def by_emulator(index: dict) -> dict[str, list[dict]]:
out: dict[str, list[dict]] = {}
for item in index.get("items", []):
@@ -60,7 +418,7 @@ def tested_item(item: dict) -> list[str]:
+ (f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" if item["flags"] else "")
+ ".",
"",
f" [Full record]({item['record_url']}), [its data]({item['data_url']}), {runs}.",
f" [Full record]({item['record_url']}), {runs}.",
"",
]
return lines
@@ -91,7 +449,7 @@ def hosts_table(index: dict) -> list[str]:
computer = item["host"].split(", load ")[0]
hosts[computer] = hosts.get(computer, 0) + 1
if not hosts:
return ["No result is recorded at the pinned commit yet.", ""]
return ["No result is recorded yet.", ""]
return [
"| Computer | Tests |",
"|----------|------:|",
@@ -108,8 +466,8 @@ def with_hosts(page: str, index: dict) -> str:
def generate_explorer_page(index: dict) -> str:
commit = index.get("commit")
repo = index.get("repo") or ""
source = (f"[retrobios-measurements]({repo}/tree/{commit}) at {_short(commit)}"
if commit else "the measurements project, not read in this build")
source = (f"[`measurements/`]({repo}/tree/{commit}/{mr.RECORDS_DIR}) at {_short(commit)}"
if commit else f"`{mr.RECORDS_DIR}/` of this repository")
return "\n".join([
"# Measurements",
"",
@@ -148,19 +506,17 @@ def data_page_section(path: Path) -> list[str]:
"## Measurements",
"",
"[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe "
"record at the pinned commit of [retrobios-measurements]"
"(https://github.com/Abdess/retrobios-measurements), and "
f"record kept in [`{mr.RECORDS_DIR}/`]({REPO}/tree/main/{mr.RECORDS_DIR}), and "
"`downloads/measurements/<emulator>.sqlite` the records of one emulator, linked "
f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads "
"[`data/measurements/index.json`](data/measurements/index.json). A record line is "
"valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json) "
"and its data file against [`core-probe-data.schema.json`](api/v1/schemas/core-probe-data.schema.json).",
"valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json).",
"",
"| Table | Columns |",
"|-------|---------|",
*(f"| `{name}` | {', '.join(f'`{c}`' for c in columns)} |" for name, columns in sqlite_tables(path)),
"",
"`meta` names the measurements commit and the record version. A list or object "
"`meta` names the commit the record links point to and the record version. A list or object "
"value is JSON text, an absent one empty. `runs.superseded` is 1 for a record a "
"later one corrects.",
"",
scripts/sync_measurements.py
-564
View File
@@ -1,564 +0,0 @@
#!/usr/bin/env python3
"""Read the measurements repository at its pin and compile what the site shows.
The records live in their own repository (Abdess/retrobios-measurements),
one append-only .jsonl per emulator. This repository holds only the pin,
`measurements-pin.json`: the repository URL and the commit read. The clone
is kept at .cache/measurements/ and read at the pinned commit with git, so
its working tree may hold records not yet committed.
From the records at the pin it writes, never commits:
public/data/measurements/index.json the compact index the explorer and
the emulator pages read, one item
per emulator and subject
public/downloads/measurements.sqlite every record, normalised
public/downloads/measurements/<emu>.sqlite one database per emulator
`--fetch` clones or updates the clone, the only step that uses the network,
and refuses a remote whose history rewrote or removed a line the pin holds.
`--bump` then moves the pin to the fetched head.
"""
from __future__ import annotations
import argparse
import json
import os
import sqlite3
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement_records as mr # noqa: E402
from artifacts import json_text, write_if_changed, write_text_atomic # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
PIN_FILE = ROOT / "measurements-pin.json"
CLONE = ROOT / ".cache" / "measurements"
INDEX_PATH = Path("data") / "measurements" / "index.json"
SQLITE_PATH = Path("downloads") / "measurements.sqlite"
PER_EMULATOR_DIR = Path("downloads") / "measurements"
SQLITE_SCHEMA_VERSION = "1"
INDEX_SCHEMA_VERSION = 1
class MeasurementsError(RuntimeError):
"""The measurements at the pin cannot be read, or break a rule."""
@dataclass(frozen=True)
class Stored:
"""One record line as read at a commit."""
emulator: str
line: int
record: dict
# --------------------------------------------------------------------------
# Pin and git
# --------------------------------------------------------------------------
def load_pin(path: Path = PIN_FILE) -> dict:
pin = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(pin.get("repo"), str) or not isinstance(pin.get("commit"), str) \
or len(pin["commit"]) != 40:
raise MeasurementsError(f"{path}: needs a repo URL and a 40-character commit")
return pin
def write_pin(pin: dict, path: Path = PIN_FILE) -> None:
write_text_atomic(str(path), json.dumps(pin, indent=2, sort_keys=True) + "\n")
def git(clone: Path, *args: str, binary: bool = False) -> str | bytes:
proc = subprocess.run(["git", "-C", str(clone), *args], capture_output=True, check=False)
if proc.returncode != 0:
raise MeasurementsError(f"git {' '.join(args)}: {proc.stderr.decode(errors='replace').strip()}")
return proc.stdout if binary else proc.stdout.decode("utf-8")
def has_commit(clone: Path, commit: str) -> bool:
if not (clone / ".git").exists():
return False
proc = subprocess.run(["git", "-C", str(clone), "cat-file", "-e", f"{commit}^{{commit}}"],
capture_output=True, check=False)
return proc.returncode == 0
def tree_files(clone: Path, commit: str) -> dict[str, str]:
"""Path -> blob id of every file under records/ and data/ at a commit."""
out: dict[str, str] = {}
listing = git(clone, "ls-tree", "-r", commit, "--", mr.RECORDS_DIR, mr.DATA_DIR)
for row in str(listing).splitlines():
meta, _, path = row.partition("\t")
out[path] = meta.split()[2]
return out
def show(clone: Path, commit: str, path: str) -> bytes:
return bytes(git(clone, "show", f"{commit}:{path}", binary=True))
# --------------------------------------------------------------------------
# Reading and checking
# --------------------------------------------------------------------------
def read_records(clone: Path, commit: str) -> list[Stored]:
"""Every record line at a commit, each checked against its rules and data."""
files = tree_files(clone, commit)
problems: list[str] = []
stored: list[Stored] = []
for path in sorted(p for p in files if p.startswith(f"{mr.RECORDS_DIR}/")):
emulator = path[len(mr.RECORDS_DIR) + 1:].removesuffix(".jsonl")
if not path.endswith(".jsonl") or "/" in emulator:
problems.append(f"{path}: records/ holds <emulator>.jsonl files only")
continue
seen: set[str] = set()
try:
records = mr.parse_lines(show(clone, commit, path).decode("utf-8"), path)
except mr.RecordError as exc:
problems.append(str(exc))
continue
for number, record in enumerate(records, 1):
where = f"{path}:{number}"
problems.extend(f"{where}: {p}" for p in mr.check_line(record))
if record.get("emulator") != emulator:
problems.append(f"{where}: emulator {record.get('emulator')!r} in {path}")
if record.get("id") in seen:
problems.append(f"{where}: id {record.get('id')} appears twice")
if record.get("supersedes") and record["supersedes"] not in seen:
problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file")
seen.add(record.get("id"))
data = mr.sidecar_path(emulator, str(record.get("id")))
if data not in files:
problems.append(f"{where}: {data} is missing")
else:
problems.extend(f"{where}: {p}" for p in mr.check_pair(record, show(clone, commit, data)))
stored.append(Stored(emulator, number, record))
if problems:
raise MeasurementsError("records at {}:\n {}".format(commit[:12], "\n ".join(problems)))
return stored
def append_only_problems(clone: Path, old: str, new: str) -> list[str]:
"""What ``new`` changed or removed of the lines and data ``old`` holds."""
before, after = tree_files(clone, old), tree_files(clone, new)
problems = []
for path, blob in sorted(before.items()):
if path not in after:
problems.append(f"{path}: removed")
elif after[path] == blob:
continue
elif path.endswith(".jsonl"):
moved = mr.appended_only(show(clone, old, path).decode("utf-8"),
show(clone, new, path).decode("utf-8"))
if moved:
problems.append(f"{path}: {moved}")
else:
problems.append(f"{path}: rewritten")
return problems
# --------------------------------------------------------------------------
# Index
# --------------------------------------------------------------------------
def host_summary(host: dict) -> str:
parts = [host.get("cpu_model") or "unknown CPU"]
if host.get("cpu_count"):
parts.append(f"{host['cpu_count']} CPUs")
parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip()
if "Linux" in host.get("os", "") else host.get("os", ""))
parts.append(f"Python {host.get('python', '')}")
before, after = host.get("load_before"), host.get("load_after")
if before and after:
parts.append(f"load {before[0]} before, {after[0]} after")
return ", ".join(part for part in parts if part)
def bench_statement(record: dict) -> str:
timing = record["timing"]
frame, fps = timing["ms_per_frame"], timing["fps"]
return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), "
f"min {frame['min']:.3f}, max {frame['max']:.3f}, over "
f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames")
def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str:
return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
def index_item(entry: Stored, runs: int, pin: dict, profiles: dict,
system_names: dict[str, str]) -> dict:
record = entry.record
profile = profiles.get(entry.emulator) or {}
check = record.get("check") or {}
core = record["core"]
content = (record["inputs"].get("content") or {})
return {
"emulator": entry.emulator,
"emulator_name": str(profile.get("emulator", entry.emulator)),
"profiled": bool(profile),
"systems": sorted(set(profile.get("systems") or [])),
"system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))],
"subject": record["subject"],
"kind": record["kind"],
"claim": check.get("claim"),
"method": check.get("method"),
"observable": check.get("observable"),
"variable": check.get("variable"),
"status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
"statement": (check.get("result") or {}).get("statement")
or (bench_statement(record) if record.get("timing") else ""),
"content": content.get("label"),
"variants": len(record["variants"]),
"repetitions": record["inputs"]["repetitions"],
"flags": len(record["flags"]),
"date": record["recorded_at"],
"host": host_summary(record["host"]),
"core": {
"library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])),
"file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"),
"commit": core.get("commit"), "build": core.get("build_command"),
},
"frontend_commit": record["frontend"].get("commit"),
"id": record["id"],
"runs": runs,
"record_url": permalink(pin["repo"], pin["commit"], mr.records_path(entry.emulator), entry.line),
"data_url": permalink(pin["repo"], pin["commit"], mr.sidecar_path(entry.emulator, record["id"])),
"sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
}
def build_index(stored: list[Stored], pin: dict | None, profiles: dict,
system_names: dict[str, str] | None = None) -> dict:
"""One item per (emulator, subject): its latest record not superseded."""
by_subject: dict[tuple[str, str], list[Stored]] = {}
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
for entry in stored:
if entry.record["id"] in live_ids:
by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry)
items = []
for key in sorted(by_subject):
entries = by_subject[key]
latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
items.append(index_item(latest, len(entries), pin or {}, profiles, system_names or {}))
items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
return {
"schema_version": INDEX_SCHEMA_VERSION,
"kind": "measurements",
"repo": (pin or {}).get("repo"),
"commit": (pin or {}).get("commit"),
"records": len(stored),
"count": len(items),
"items": items,
}
# --------------------------------------------------------------------------
# SQLite
# --------------------------------------------------------------------------
SCHEMA_SQL = """
PRAGMA journal_mode = OFF;
PRAGMA synchronous = OFF;
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE cores (
sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT,
build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT
);
CREATE TABLE hosts (
id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL,
kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL
);
CREATE TABLE emulator_systems (
emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system)
);
CREATE TABLE runs (
id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL,
kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT,
superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
script_modified INTEGER, flags TEXT NOT NULL, data_sha256 TEXT NOT NULL,
data_bytes INTEGER NOT NULL, record_url TEXT NOT NULL, line INTEGER NOT NULL,
migrated_from_version INTEGER
);
CREATE TABLE inputs (
run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT,
content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL,
warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL,
observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL
);
CREATE TABLE content_tracks (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL,
PRIMARY KEY (run_id, position)
);
CREATE TABLE checks (
run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL,
observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL,
statement TEXT NOT NULL
);
CREATE TABLE check_groups (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position)
);
CREATE TABLE variants (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
);
CREATE TABLE system_files (
run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
destination TEXT NOT NULL, sha1 TEXT,
PRIMARY KEY (run_id, variant_position, destination)
);
CREATE TABLE benchmarks (
run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL,
median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL,
min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric)
);
CREATE INDEX runs_emulator_idx ON runs(emulator, subject);
CREATE INDEX runs_subject_idx ON runs(subject);
CREATE INDEX emulator_systems_system_idx ON emulator_systems(system);
CREATE INDEX checks_claim_idx ON checks(claim);
CREATE INDEX system_files_sha1_idx ON system_files(sha1);
"""
TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks",
"checks", "check_groups", "variants", "system_files", "benchmarks")
def _host_key(host: dict) -> tuple:
return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel",
"machine", "python"))
def write_sqlite(destination: Path, stored: list[Stored], pin: dict | None, profiles: dict,
scratch: Path) -> None:
"""A deterministic database of the records: sorted rows, no clock."""
scratch.mkdir(parents=True, exist_ok=True)
temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite"
temp.unlink(missing_ok=True)
destination.parent.mkdir(parents=True, exist_ok=True)
stored = sorted(stored, key=lambda s: (s.emulator, s.line))
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)}
connection = sqlite3.connect(temp)
try:
connection.executescript(SCHEMA_SQL)
meta = {
"schema_version": SQLITE_SCHEMA_VERSION,
"record_version": str(mr.RECORD_VERSION),
"measurements_repo": (pin or {}).get("repo") or "",
"measurements_commit": (pin or {}).get("commit") or "",
"license": "MIT",
"scope": "core probe records: the record lines, not their data files",
}
connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [
(sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")),
json_text(core.get("build_command")), json_text(core.get("compiler")),
json_text(core.get("library_name")), json_text(core.get("library_version")))
for sha, core in sorted(cores.items())])
connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [
(number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])])
emulators = sorted({s.emulator for s in stored})
connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [
(emu, system) for emu in emulators
for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))])
for entry in stored:
_insert_run(connection, entry, pin or {}, hosts, entry.record["id"] not in live_ids)
connection.commit()
connection.execute("VACUUM")
finally:
connection.close()
os.replace(temp, destination)
def _insert_run(connection: sqlite3.Connection, entry: Stored, pin: dict,
hosts: dict, superseded: bool) -> None:
r = entry.record
host, front, inputs = r["host"], r["frontend"], r["inputs"]
rid = r["id"]
connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
front.get("commit"), front["script_sha256"],
None if front.get("script_modified") is None else int(front["script_modified"]),
json_text(r["flags"]), r["data"]["sha256"], r["data"]["bytes"],
permalink(pin.get("repo", ""), pin.get("commit", ""), mr.records_path(entry.emulator), entry.line),
entry.line, r.get("migrated_from_version")))
content = inputs.get("content") or {}
connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, content.get("label"), content.get("sha1"), content.get("size"),
json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"],
inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]),
inputs["frame_hash_every"], inputs["joypad_ports"]))
connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [
(rid, position, track["file"], track["sha1"], track["size"])
for position, track in enumerate(content.get("tracks") or [])])
check = r.get("check")
if check:
result = check["result"]
connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, check["claim"], check["method"], check["observable"], check["variable"],
result["status"], result["statement"]))
connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
(rid, position, json_text(group["value"]) if "value" in group else None,
group.get("value_sha1"), json_text(group["variants"]))
for position, group in enumerate(result["groups"])])
for position, variant in enumerate(r["variants"]):
connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, position, variant["label"], variant["repetitions"], int(variant["loaded"]),
json_text(variant["flags"]), json_text(variant["observables"]),
json_text(variant["files_opened"]), json_text(variant["files_probed_absent"]),
json_text(variant["tracing"])))
connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
(rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
for metric, stats in sorted((r.get("timing") or {}).items()):
connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"]))
# --------------------------------------------------------------------------
# Building the site's files
# --------------------------------------------------------------------------
@dataclass
class Built:
index: dict
paths: list[Path]
def load_profiles(emulators_dir: Path) -> dict:
from common import load_emulator_profiles # noqa: PLC0415
return load_emulator_profiles(str(emulators_dir), skip_aliases=False)
def records_at_pin(clone: Path, pin: dict, require: bool) -> list[Stored] | None:
"""The records at the pin, or None when the clone does not hold it."""
if has_commit(clone, pin["commit"]):
return read_records(clone, pin["commit"])
message = (f"measurements commit {pin['commit'][:12]} is not in {clone}: run "
f"python scripts/sync_measurements.py --fetch")
if require:
raise MeasurementsError(message)
print(f"WARNING: {message}. The site lists no measurements.", file=sys.stderr)
return None
def build(public: Path, clone: Path = CLONE, pin_path: Path = PIN_FILE,
emulators_dir: Path = ROOT / "emulators", require: bool = False,
scratch: Path = ROOT / "tmp" / "site",
system_names: dict[str, str] | None = None) -> Built:
"""Write the index and the databases under the site's public folder."""
pin = load_pin(pin_path)
stored = records_at_pin(clone, pin, require)
profiles = load_profiles(emulators_dir)
if stored is None:
pin = None
index = build_index(stored or [], pin, profiles, system_names)
paths = [public / INDEX_PATH]
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
write_if_changed(str(public / INDEX_PATH),
json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
write_sqlite(public / SQLITE_PATH, stored or [], pin, profiles, scratch)
paths.append(public / SQLITE_PATH)
for emulator in sorted({s.emulator for s in stored or []}):
target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite"
write_sqlite(target, [s for s in stored or [] if s.emulator == emulator], pin, profiles, scratch)
paths.append(target)
return Built(index, paths)
def fetch(clone: Path, pin: dict) -> str:
"""Clone or update the measurements repository. Returns the remote head."""
if not (clone / ".git").exists():
clone.parent.mkdir(parents=True, exist_ok=True)
proc = subprocess.run(["git", "clone", "--quiet", pin["repo"], str(clone)],
capture_output=True, text=True, check=False)
if proc.returncode != 0:
raise MeasurementsError(f"git clone {pin['repo']}: {proc.stderr.strip()}")
else:
git(clone, "fetch", "--quiet", "origin")
head = str(git(clone, "rev-parse", "refs/remotes/origin/HEAD")).strip()
branch = str(git(clone, "rev-parse", "--abbrev-ref", "HEAD")).strip()
if branch != "HEAD":
proc = subprocess.run(["git", "-C", str(clone), "merge", "--ff-only", "--quiet", head],
capture_output=True, text=True, check=False)
if proc.returncode != 0:
print(f"WARNING: {clone} has local commits, its branch was left as is: "
f"{proc.stderr.strip()}", file=sys.stderr)
return head
def remote_head(repo: str) -> str:
proc = subprocess.run(["git", "ls-remote", repo, "HEAD"], capture_output=True, text=True,
check=False, timeout=60)
if proc.returncode != 0 or not proc.stdout.strip():
raise MeasurementsError(f"git ls-remote {repo}: {proc.stderr.strip() or 'no HEAD'}")
return proc.stdout.split()[0]
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
parser.add_argument("--fetch", action="store_true",
help="clone or update the clone, and check the remote kept every line the pin holds")
parser.add_argument("--bump", action="store_true",
help=f"after --fetch, move {PIN_FILE.name} to the fetched head")
parser.add_argument("--check-only", action="store_true",
help="check the records at the pin and write nothing")
parser.add_argument("--clone", default=str(CLONE), help="the clone (default: %(default)s)")
parser.add_argument("--pin", default=str(PIN_FILE), help="the pin file (default: %(default)s)")
parser.add_argument("--site-dir", default=str(ROOT / "website"),
help="the site project whose public/ folder is written (default: %(default)s)")
args = parser.parse_args(argv)
if args.bump and not args.fetch:
parser.error("--bump needs --fetch")
clone, pin_path = Path(args.clone), Path(args.pin)
try:
pin = load_pin(pin_path)
if args.fetch:
head = fetch(clone, pin)
problems = append_only_problems(clone, pin["commit"], head) \
if has_commit(clone, pin["commit"]) else [f"pin {pin['commit']} is not in the remote history"]
if problems:
print("ERROR: the remote changed records the pin holds:\n " + "\n ".join(problems),
file=sys.stderr)
return 1
records = read_records(clone, head)
pinned = len(read_records(clone, pin["commit"]))
print(f"{head[:12]}: {len(records)} records, {len(records) - pinned} after the pin "
f"{pin['commit'][:12]}")
if args.bump and head != pin["commit"]:
write_pin({**pin, "commit": head}, pin_path)
print(f"pin moved to {head}")
return 0
if args.check_only:
stored = records_at_pin(clone, pin, require=True)
print(f"{pin['commit'][:12]}: {len(stored or [])} records, every line and data file checked")
return 0
built = build(Path(args.site_dir) / "public", clone, pin_path, require=True)
print(f"{built.index['records']} records, {built.index['count']} subjects: "
+ ", ".join(str(p.relative_to(Path(args.site_dir))) for p in built.paths))
return 0
except MeasurementsError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
from common import run_cli
run_cli(main)
scripts/validate_schemas.py
+14 -31
View File
@@ -593,46 +593,29 @@ def _format_errors() -> list[str]:
]
def _measurement_errors(clone: Path | None = None, pin_path: Path | None = None) -> list[str]:
"""Every record line and data file at the measurements pin, against their
schemas and the rules sync_measurements.py applies.
The records live in their own repository. Without a clone holding the
pin there is nothing to read, and the run says so.
"""
import sync_measurements # noqa: PLC0415
def _measurement_errors(root: Path | None = None) -> list[str]:
"""Every record line under measurements/, against its schema and the
rules measurement_records.read_records applies."""
import measurement_records as mr # noqa: PLC0415
clone = clone or sync_measurements.CLONE
pin = sync_measurements.load_pin(pin_path or sync_measurements.PIN_FILE)
if not sync_measurements.has_commit(clone, pin["commit"]):
print(f"SKIP measurements: {pin['commit'][:12]} is not in {clone} "
"(python scripts/sync_measurements.py --fetch)")
return []
commit = pin["commit"]
record_validator = _validator("core-probe-record.schema.json")
data_validator = _validator("core-probe-data.schema.json")
root = root or ROOT
validator = _validator("core-probe-record.schema.json")
out: list[str] = []
files = sync_measurements.tree_files(clone, commit)
for path in sorted(p for p in files if p.endswith(".jsonl")):
text = sync_measurements.show(clone, commit, path).decode("utf-8")
for number, line in enumerate(text.splitlines(), 1):
for path, file in sorted(mr.tree_files(root).items()):
if not path.endswith(".jsonl"):
continue
for number, line in enumerate(file.read_text(encoding="utf-8").splitlines(), 1):
try:
record = json.loads(line)
except json.JSONDecodeError as exc:
out.append(f"measurements {path}:{number}: {exc}")
out.append(f"{path}:{number}: {exc}")
continue
out.extend(_errors(record_validator, record, f"measurements {path}:{number}"))
data = mr.sidecar_path(path[len("records/"):-len(".jsonl")], str(record.get("id")))
if data in files:
out.extend(_errors(data_validator,
mr.read_sidecar(sync_measurements.show(clone, commit, data)),
f"measurements {data}"))
out.extend(_errors(validator, record, f"{path}:{number}"))
if not out:
try:
sync_measurements.read_records(clone, commit)
except sync_measurements.MeasurementsError as exc:
out.append(f"measurements {exc}")
mr.read_records(root)
except mr.MeasurementsError as exc:
out.append(str(exc))
return out