kmoody/libretro
1
0 Fork 0
mirror of https://github.com/Abdess/retroarch_system.git synced 2026-11-01 00:27:20 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor: read measurements from the local tree

Browse Source 
The site compiles the index and the SQLite files from measurements/
instead of a pinned clone. The pin, the sync script, the CI clone steps,
the freshness area and its refresh handler are gone. core_probe --record
writes into measurements/ of the checkout by default, and a test fails
when a line or data file HEAD holds changed or vanished.
This commit is contained in:
Abdessamad Derraz committed 2026-10-27 09:11:12 +01:00
1 parent 7dca6ae052
commit 571e50f2a8
17 files changed
+699 -881
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/deploy-site.yml
+2 -15
View File
@@ -57,7 +57,6 @@ on:
- "scripts/slots.py"
- "scripts/sourceref.py"
- "scripts/split_pack.py"
- "scripts/sync_measurements.py"
- "scripts/truth.py"
- "scripts/upstream.py"
- "scripts/validate_schemas.py"
@@ -72,7 +71,7 @@ on:
- "database.json"
- "release.json"
- "derivable.json"
- "measurements-pin.json"
- "measurements/**"
# The Starlight project: config, components, styles, lockfile. Its
# generated folders are gitignored and never trigger anything.
- "website/**"
@@ -118,18 +117,6 @@ jobs:
- run: npm ci --prefix website
# The core probe records live in Abdess/retrobios-measurements, read
# at the commit measurements-pin.json names and never at its head: a
# record reaches the site when the pin moves. One shallow fetch of that
# commit, with git alone.
- name: Clone the measurements at their pin
run: |
repo=$(python -c "import json; print(json.load(open('measurements-pin.json'))['repo'])")
commit=$(python -c "import json; print(json.load(open('measurements-pin.json'))['commit'])")
git init -q .cache/measurements
git -C .cache/measurements fetch -q --depth 1 "$repo" "$commit"
git -C .cache/measurements checkout -q --detach "$commit"
- name: Validate data contracts
run: python scripts/validate_schemas.py
@@ -149,7 +136,7 @@ jobs:
- name: Generate site
run: |
python scripts/generate_site.py --require-measurements
python scripts/generate_site.py
python scripts/generate_readme.py --db database.json --platforms-dir platforms
# starlight-links-validator fails the build on a dead link or anchor.
npm run build --prefix website
.github/workflows/validate.yml
+3 -13
View File
@@ -29,7 +29,7 @@ on:
- "database.json"
- "release.json"
- "derivable.json"
- "measurements-pin.json"
- "measurements/**"
- "website/**"
# Spelled out twice because the workflow parser reads no YAML anchor. A test
# holds the two lists equal.
@@ -54,7 +54,7 @@ on:
- "database.json"
- "release.json"
- "derivable.json"
- "measurements-pin.json"
- "measurements/**"
- "website/**"
permissions:
@@ -259,21 +259,11 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# The measurement records at their pin, as deploy-site.yml reads them:
# the pages and the schema check read the same lines.
- name: Clone the measurements at their pin
run: |
repo=$(python -c "import json; print(json.load(open('measurements-pin.json'))['repo'])")
commit=$(python -c "import json; print(json.load(open('measurements-pin.json'))['commit'])")
git init -q .cache/measurements
git -C .cache/measurements fetch -q --depth 1 "$repo" "$commit"
git -C .cache/measurements checkout -q --detach "$commit"
# The pages are gitignored, and the tests that read them fail
# in CI without it. The suite runs whatever this step returned, so a
# generator that stops still leaves every other test reporting.
- name: Generate the site
run: python scripts/generate_site.py --require-measurements
run: python scripts/generate_site.py
- name: Run test suite
if: ${{ !cancelled() }}
measurements-pin.json
-4
View File
@@ -1,4 +0,0 @@
{
"commit": "75e878143687201d1d7ca34f09cb43be3dc8d855",
"repo": "https://github.com/Abdess/retrobios-measurements"
}
schemas/core-probe-data.schema.json
+1 -1
View File
@@ -2,7 +2,7 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-data.schema.json",
"title": "RetroBIOS core probe record data",
"description": "The sidecar of a core probe record, data/<emulator>/<id>.json.gz: every repetition in full, the raw timing values and compared values too large for the record line. With the record it rebuilds the full result.",
"description": "The sidecar of a core probe record, measurements/data/<emulator>/<id>.json.gz: every repetition in full, the raw timing values and compared values too large for the record line. With the record it rebuilds the full result.",
"type": "object",
"additionalProperties": false,
"required": [
schemas/core-probe-record.schema.json
+2 -2
View File
@@ -2,7 +2,7 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-record.schema.json",
"title": "RetroBIOS core probe record",
"description": "One line of records/<emulator>.jsonl in the measurements repository: one run of scripts/core_probe.py, a libretro core executed headless, with what is needed to judge and reproduce it. The bulky rest (every repetition, core logs, raw timings) is the gzip sidecar data/<emulator>/<id>.json.gz, valid against core-probe-data.schema.json, whose sha256 is in `data`.",
"description": "One line of measurements/<emulator>.jsonl: one run of scripts/core_probe.py, a libretro core executed headless, with what is needed to judge and reproduce it. The bulky rest (every repetition, core logs, raw timings) is the gzip sidecar measurements/data/<emulator>/<id>.json.gz, valid against core-probe-data.schema.json, whose sha256 is in `data`.",
"type": "object",
"additionalProperties": false,
"required": [
@@ -32,7 +32,7 @@
},
"emulator": {
"$ref": "#/$defs/slug",
"description": "The emulator profile the record is filed under: records/<emulator>.jsonl."
"description": "The emulator profile the record is filed under: measurements/<emulator>.jsonl."
},
"subject": {
"$ref": "#/$defs/slug",
scripts/check_freshness.py
+3 -31
View File
@@ -3,7 +3,7 @@
Each layer the repository transcribes moves on its own schedule: platform
lists, hardware targets, core-info, buildbot system assets, dump catalogs,
romset DATs, the measurements pin, the CI toolchain. This script asks each of them the same
romset DATs, the CI toolchain. This script asks each of them the same
question, is the local copy the one upstream serves, and answers with one
row per subject.
@@ -40,7 +40,7 @@ import shutil
import subprocess
import sys
import tempfile
from collections.abc import Callable, Iterator
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass, field
from datetime import datetime, timezone
@@ -80,8 +80,7 @@ INSTALLERS = ("install.sh", "install.ps1")
SHOWN_ITEMS = 4 # items named in a diff summary before ", ..."
SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..."
AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "measurements", "ci",
"profiles")
AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "ci", "profiles")
OK, STALE, UNKNOWN, ERROR, SKIPPED = "OK", "STALE", "UNKNOWN", "ERROR", "SKIPPED"
# Every status a finding may carry, in the order the summary counts them.
STATUSES = (STALE, UNKNOWN, ERROR, SKIPPED, OK)
@@ -915,31 +914,6 @@ def _check_buildbot_system() -> Finding:
detail=f"{names}{more} (_data_dirs.yml)", items=every)
# --- measurements -----------------------------------------------------------
def check_measurements(offline: bool, head: Callable[[str], str] | None = None,
pin_path: Path | None = None) -> Finding:
"""The measurements pin against the head of the measurements repository.
A record added there reaches the site only once the pin moves to it.
"""
import sync_measurements # noqa: PLC0415
pin = sync_measurements.load_pin(pin_path or sync_measurements.PIN_FILE)
local = pin["commit"][:12]
if offline:
return Finding("measurements", "records pin", SKIPPED, local=local, detail="offline")
try:
remote = (head or sync_measurements.remote_head)(pin["repo"])
except (sync_measurements.MeasurementsError, OSError, subprocess.TimeoutExpired) as exc:
return Finding("measurements", "records pin", UNKNOWN, local=local, detail=str(exc))
if remote == pin["commit"]:
return Finding("measurements", "records pin", OK, local=local, remote=remote[:12])
return Finding("measurements", "records pin", STALE, local=local, remote=remote[:12],
detail="newer records: python scripts/sync_measurements.py --fetch --bump")
# --- catalogs and recipes ---------------------------------------------------
@@ -1532,8 +1506,6 @@ def main() -> int:
findings.extend(check_data(args.offline))
if "catalogs" in areas:
findings.extend(check_catalogs(args.cache_dir, args.offline))
if "measurements" in areas:
findings.append(check_measurements(args.offline))
if "ci" in areas:
findings.extend(check_ci(args.cache_dir, args.offline))
if "profiles" in areas:
scripts/core_probe.py
+8 -36
View File
@@ -10,10 +10,10 @@ Three modes share one runner:
Each repetition runs in its own process, under strace when it is installed,
so the files a core opened are read from the kernel rather than guessed.
`--record DIR` appends one record per invocation to a clone of the
measurements repository, carrying what is needed to reproduce it (core build,
frontend commit, host, inputs by hash). measurement_records.py splits it into
a slim line and a gzip sidecar.
`--record` appends one record per invocation to measurements/ of this
repository, carrying what is needed to reproduce it (core build, frontend
commit, host, inputs by hash). measurement_records.py splits it into a slim
line and a gzip sidecar.
"""
from __future__ import annotations
@@ -39,7 +39,6 @@ from dataclasses import dataclass, field
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
MEASUREMENTS_CLONE = ROOT / ".cache" / "measurements"
MIN_BENCH_REPETITIONS = 5
DEFAULT_WORK_DIR = ROOT / "tmp" / "core_probe"
BASE_OBSERVABLES = ("frame_sha1", "video_sha1", "audio_sha1", "state_sha1")
@@ -771,10 +770,10 @@ def build_parser() -> argparse.ArgumentParser:
common.add_argument("--core-repo", help="source repository URL of the core build")
common.add_argument("--core-commit", help="commit the core was built from")
common.add_argument("--build-command", help="command and flags that built the core")
common.add_argument("--record", metavar="DIR",
help="append the record to the measurements clone DIR "
f"(records/<emulator>.jsonl and data/<emulator>/), "
f"for example {MEASUREMENTS_CLONE.relative_to(ROOT)}")
common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
help="append the record to measurements/<emulator>.jsonl and write "
"its data file in measurements/data/<emulator>/ of the checkout "
"DIR (default: this one)")
common.add_argument("--emulator", help="profile the record is filed under (default: the "
"profile whose cores: names the .so, else the "
"core's library name)")
@@ -823,7 +822,6 @@ def validate_args(args: argparse.Namespace) -> list[str]:
if args.png_dir and not args.frame_hash_every:
problems.append("--png-dir needs --frame-hash-every")
if args.record:
problems.extend(record_target_problems(Path(args.record)))
if not args.emulator and Path(args.core).is_file():
try:
default_emulator(Path(args.core), None)
@@ -907,32 +905,6 @@ def check_result(variants: list[dict], observable: str) -> dict:
}
def record_target_problems(directory: Path) -> list[str]:
"""Why records may not be written under ``directory``.
Records live in their own repository. A directory inside this one's
tracked tree would put them in its history, megabytes per campaign;
an ignored directory (tmp/, the clone under .cache/) is fine.
"""
target = directory.resolve()
if target != ROOT and ROOT not in target.parents:
return []
try:
ignored = subprocess.run(["git", "check-ignore", "-q", str(target)], cwd=ROOT,
capture_output=True, check=False).returncode
except OSError as exc:
return [f"--record {directory}: cannot ask git whether it is tracked ({exc})"]
if ignored == 0:
return []
if ignored != 1:
# 128: not a git checkout, nothing to keep records out of.
return []
return [f"--record {directory}: inside the tracked tree of this repository. Records go "
f"to the measurements repository: clone it with "
f"`python scripts/sync_measurements.py --fetch` and pass "
f"--record {MEASUREMENTS_CLONE.relative_to(ROOT)}"]
def default_emulator(core_file: Path, library_name: str | None) -> str:
"""The profile whose cores: names the .so, else the library name as a slug."""
stem = core_file.name.split(".")[0].removesuffix("_libretro")
scripts/generate_site.py
+4 -10
View File
@@ -110,7 +110,6 @@ from siteguides import (
)
import release_record
import sitemeasurements
import sync_measurements
# The Starlight project, and where its pages go inside it.
STARLIGHT_DIR = "website"
@@ -11007,11 +11006,6 @@ def main() -> None:
help="directory the pages are written to (default: the project's "
"content collection, src/content/docs)",
)
parser.add_argument(
"--require-measurements", action="store_true",
help="fail when the measurements clone does not hold the pinned commit "
"(the deploy build passes it; without it the site lists no measurements)",
)
args = parser.parse_args()
if args.docs_dir is None:
args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT)
@@ -11168,9 +11162,9 @@ def _generate(args: argparse.Namespace) -> None:
if clash:
raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}")
print("Compiling the measurements at their pin...")
measured = sync_measurements.build(
public, emulators_dir=Path(args.emulators_dir), require=args.require_measurements,
print("Compiling the measurements...")
measured = sitemeasurements.build(
public, emulators_dir=Path(args.emulators_dir), repo=REPO_URL,
system_names={sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))},
)
for path in measured.paths:
@@ -11187,7 +11181,7 @@ def _generate(args: argparse.Namespace) -> None:
)
write_release_advertised(Path(args.site_dir) / "src" / "generated")
write_if_changed(str(docs / "data.md"),
generate_data_page(exports, public / sync_measurements.SQLITE_PATH)
generate_data_page(exports, public / sitemeasurements.SQLITE_PATH)
+ reference_next())
PAGE_DATA["data.md"] = dataset_structured_data(exports, db)
scripts/measurement_records.py
+119 -7
View File
@@ -1,9 +1,9 @@
"""Core probe records as the measurements repository stores them.
"""Core probe records as this repository stores them.
A run of core_probe.py produces one full record. It is stored as two parts:
records/<emulator>.jsonl one slim record per line, append-only
data/<emulator>/<id>.json.gz the bulky rest, named by the record id
measurements/<emulator>.jsonl one slim record per line, append-only
measurements/data/<emulator>/<id>.json.gz the bulky rest, named by the record id
The slim record keeps what a reader needs to judge and reproduce the result:
claim, method, observable, result, statistics, host, inputs by hash, core
@@ -18,7 +18,7 @@ A record id is the date and the first 12 hex digits of the sha256 of the
record without its id. A record carried over from the first layout keeps
the id it had, which hashes that full record.
Standard library only: core_probe.py, sync_measurements.py and the site
Standard library only: core_probe.py, validate_schemas.py and the site
read it.
"""
@@ -29,12 +29,14 @@ import gzip
import hashlib
import json
import re
import subprocess
from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path
RECORD_VERSION = 2
MEASUREMENTS_REPO = "https://github.com/Abdess/retrobios-measurements"
RECORDS_DIR = "records"
DATA_DIR = "data"
RECORDS_DIR = "measurements"
DATA_DIR = f"{RECORDS_DIR}/data"
# Fields a repetition keeps in the slim record, per variant, from the first one.
VARIANT_SUMMARY = ("observables", "files_opened", "files_probed_absent", "tracing")
# Observables too large for a line: their values stay in the sidecar.
@@ -244,3 +246,113 @@ def live(records: Iterable[dict]) -> list[dict]:
records = list(records)
superseded = {r["supersedes"] for r in records if r.get("supersedes")}
return [r for r in records if r["id"] not in superseded]
class MeasurementsError(RuntimeError):
"""The records of a tree cannot be read, or break a rule."""
@dataclass(frozen=True)
class Stored:
"""One record line as read from the tree."""
emulator: str
line: int
record: dict
def tree_files(root: Path) -> dict[str, Path]:
"""Repository path -> file of every record file and data file under ``root``."""
base = root / RECORDS_DIR
if not base.is_dir():
return {}
return {path.relative_to(root).as_posix(): path
for path in sorted(base.rglob("*")) if path.is_file()}
def read_records(root: Path) -> list[Stored]:
"""Every record line under ``root``, each checked against its rules and data."""
files = tree_files(root)
data_prefix = f"{DATA_DIR}/"
problems: list[str] = []
stored: list[Stored] = []
claimed: set[str] = set()
for path in sorted(p for p in files if not p.startswith(data_prefix)):
emulator = path[len(RECORDS_DIR) + 1:].removesuffix(".jsonl")
if not path.endswith(".jsonl") or "/" in emulator:
problems.append(f"{path}: {RECORDS_DIR}/ holds <emulator>.jsonl files and data/ only")
continue
seen: set[str] = set()
try:
records = parse_lines(files[path].read_text(encoding="utf-8"), path)
except RecordError as exc:
problems.append(str(exc))
continue
for number, record in enumerate(records, 1):
where = f"{path}:{number}"
problems.extend(f"{where}: {p}" for p in check_line(record))
if record.get("emulator") != emulator:
problems.append(f"{where}: emulator {record.get('emulator')!r} in {path}")
if record.get("id") in seen:
problems.append(f"{where}: id {record.get('id')} appears twice")
if record.get("supersedes") and record["supersedes"] not in seen:
problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file")
seen.add(record.get("id"))
data = sidecar_path(emulator, str(record.get("id")))
claimed.add(data)
if data not in files:
problems.append(f"{where}: {data} is missing")
else:
problems.extend(f"{where}: {p}" for p in check_pair(record, files[data].read_bytes()))
stored.append(Stored(emulator, number, record))
problems.extend(f"{path}: no record line names it"
for path in sorted(files) if path.startswith(data_prefix) and path not in claimed)
if problems:
raise MeasurementsError("records under {}:\n {}".format(root, "\n ".join(problems)))
return stored
def _git(root: Path, *args: str) -> subprocess.CompletedProcess:
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False)
def committed_files(root: Path, ref: str) -> dict[str, bytes]:
"""Path -> content of every record and data file at ``ref``."""
listing = _git(root, "ls-tree", "-r", "-z", "--name-only", ref, "--", RECORDS_DIR)
if listing.returncode != 0:
raise MeasurementsError(f"git ls-tree {ref}: {listing.stderr.decode(errors='replace').strip()}")
out = {}
for path in filter(None, listing.stdout.decode("utf-8").split("\0")):
blob = _git(root, "show", f"{ref}:{path}")
if blob.returncode != 0:
raise MeasurementsError(f"git show {ref}:{path}: {blob.stderr.decode(errors='replace').strip()}")
out[path] = blob.stdout
return out
def append_only_problems(root: Path, ref: str = "HEAD") -> list[str]:
"""What the tree under ``root`` changed or removed of the lines and data
files ``ref`` holds. Lines added at the end and new files are allowed."""
before, after = committed_files(root, ref), tree_files(root)
problems = []
for path, content in sorted(before.items()):
if path not in after:
problems.append(f"{path}: removed")
continue
now = after[path].read_bytes()
if now == content:
continue
if path.endswith(".jsonl"):
moved = appended_only(content.decode("utf-8"), now.decode("utf-8"))
if moved:
problems.append(f"{path}: {moved}")
else:
problems.append(f"{path}: rewritten")
return problems
def last_commit(root: Path) -> str | None:
"""The last commit that touched the records: the one their permalinks name."""
proc = _git(root, "log", "-1", "--format=%H", "--", RECORDS_DIR)
commit = proc.stdout.decode().strip() if proc.returncode == 0 else ""
return commit if re.fullmatch(r"[0-9a-f]{40}", commit) else None
scripts/refresh_stale.py
+1 -9
View File
@@ -27,9 +27,6 @@ Mapping (stale -> command):
--source mame --fetch
catalogs/fbneo recipes python -m scripts.scraper.romset_dat_importer
--source fbneo --fetch
measurements/records pin
python scripts/sync_measurements.py --fetch --bump,
which refuses a remote that rewrote a pinned line
A platform that inherits from a rescraped one (retropie and lakka from
retroarch) reads the parent's file: its jobs start once the parent's chain
@@ -66,7 +63,7 @@ CHECK_FRESHNESS = REPO_ROOT / "scripts" / "check_freshness.py"
PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml"
DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml"
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs", "measurements")
AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs")
MANUAL_AREAS = ("coreinfo", "ci", "profiles")
JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10
@@ -410,11 +407,6 @@ def _command_for(
# redump / no-intro / tosec need manual DAT download.
return None
if area == "measurements":
# --fetch refuses a remote that rewrote a pinned line, and --bump
# only rewrites measurements-pin.json: nothing is committed.
return [sys.executable, "scripts/sync_measurements.py", "--fetch", "--bump"]
# coreinfo, ci, profiles: never auto-refresh.
return None
scripts/sitemeasurements.py
+360 -10
View File
@@ -1,24 +1,375 @@
"""The site's measurement pages, from the index sync_measurements.py writes.
"""The measurements the site shows, from the records under measurements/.
Three places read the same index: the Tested block of each emulator page,
the Measurements explorer page and the host table of the methodology page.
A number is never shown without its method and host beside it.
The records are core probe runs, one append-only .jsonl per emulator with a
gzip data file per record (measurement_records.py). From them the build
writes, never committed:
public/data/measurements/index.json the compact index the explorer and
the emulator pages read, one item
per emulator and subject
public/downloads/measurements.sqlite every record, normalised
public/downloads/measurements/<emu>.sqlite one database per emulator
Three places read the index: the Tested block of each emulator page, the
Measurements explorer page and the host table of the methodology page. A
number is never shown without its method and host beside it. A record links
its line at the last commit that touched measurements/.
"""
from __future__ import annotations
import json
import os
import re
import sqlite3
import sys
from dataclasses import dataclass
from pathlib import Path
import measurement_records as mr
import sitemarkup as markup
from artifacts import json_text, write_if_changed
from siterender import _escape_tags
ROOT = Path(__file__).resolve().parent.parent
REPO = "https://github.com/Abdess/retrobios"
EXPLORER_PAGE = "measurements.md"
METHOD_PAGE = "wiki/measurements.md"
HOSTS_MARKER = "<!-- measurements-hosts -->"
INDEX_PATH = Path("data") / "measurements" / "index.json"
SQLITE_PATH = Path("downloads") / "measurements.sqlite"
PER_EMULATOR_DIR = Path("downloads") / "measurements"
SQLITE_SCHEMA_VERSION = "1"
INDEX_SCHEMA_VERSION = 1
# --------------------------------------------------------------------------
# Index
# --------------------------------------------------------------------------
def host_summary(host: dict) -> str:
parts = [host.get("cpu_model") or "unknown CPU"]
if host.get("cpu_count"):
parts.append(f"{host['cpu_count']} CPUs")
parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip()
if "Linux" in host.get("os", "") else host.get("os", ""))
parts.append(f"Python {host.get('python', '')}")
before, after = host.get("load_before"), host.get("load_after")
if before and after:
parts.append(f"load {before[0]} before, {after[0]} after")
return ", ".join(part for part in parts if part)
def bench_statement(record: dict) -> str:
timing = record["timing"]
frame, fps = timing["ms_per_frame"], timing["fps"]
return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), "
f"min {frame['min']:.3f}, max {frame['max']:.3f}, over "
f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames")
def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str:
return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
system_names: dict[str, str]) -> dict:
record = entry.record
profile = profiles.get(entry.emulator) or {}
check = record.get("check") or {}
core = record["core"]
content = (record["inputs"].get("content") or {})
return {
"emulator": entry.emulator,
"emulator_name": str(profile.get("emulator", entry.emulator)),
"profiled": bool(profile),
"systems": sorted(set(profile.get("systems") or [])),
"system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))],
"subject": record["subject"],
"kind": record["kind"],
"claim": check.get("claim"),
"method": check.get("method"),
"observable": check.get("observable"),
"variable": check.get("variable"),
"status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
"statement": (check.get("result") or {}).get("statement")
or (bench_statement(record) if record.get("timing") else ""),
"content": content.get("label"),
"variants": len(record["variants"]),
"repetitions": record["inputs"]["repetitions"],
"flags": len(record["flags"]),
"date": record["recorded_at"],
"host": host_summary(record["host"]),
"core": {
"library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])),
"file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"),
"commit": core.get("commit"), "build": core.get("build_command"),
},
"frontend_commit": record["frontend"].get("commit"),
"id": record["id"],
"runs": runs,
"record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line),
"data_url": permalink(source["repo"], source["commit"], mr.sidecar_path(entry.emulator, record["id"])),
"sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
}
def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
system_names: dict[str, str] | None = None) -> dict:
"""One item per (emulator, subject): its latest record not superseded."""
by_subject: dict[tuple[str, str], list[mr.Stored]] = {}
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
for entry in stored:
if entry.record["id"] in live_ids:
by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry)
items = []
for key in sorted(by_subject):
entries = by_subject[key]
latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
items.append(index_item(latest, len(entries), source or {}, profiles, system_names or {}))
items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
return {
"schema_version": INDEX_SCHEMA_VERSION,
"kind": "measurements",
"repo": (source or {}).get("repo"),
"commit": (source or {}).get("commit"),
"records": len(stored),
"count": len(items),
"items": items,
}
# --------------------------------------------------------------------------
# SQLite
# --------------------------------------------------------------------------
SCHEMA_SQL = """
PRAGMA journal_mode = OFF;
PRAGMA synchronous = OFF;
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE cores (
sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT,
build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT
);
CREATE TABLE hosts (
id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL,
kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL
);
CREATE TABLE emulator_systems (
emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system)
);
CREATE TABLE runs (
id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL,
kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT,
superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
script_modified INTEGER, flags TEXT NOT NULL, data_sha256 TEXT NOT NULL,
data_bytes INTEGER NOT NULL, record_url TEXT NOT NULL, line INTEGER NOT NULL,
migrated_from_version INTEGER
);
CREATE TABLE inputs (
run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT,
content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL,
warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL,
observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL
);
CREATE TABLE content_tracks (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL,
PRIMARY KEY (run_id, position)
);
CREATE TABLE checks (
run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL,
observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL,
statement TEXT NOT NULL
);
CREATE TABLE check_groups (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position)
);
CREATE TABLE variants (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
);
CREATE TABLE system_files (
run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
destination TEXT NOT NULL, sha1 TEXT,
PRIMARY KEY (run_id, variant_position, destination)
);
CREATE TABLE benchmarks (
run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL,
median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL,
min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric)
);
CREATE INDEX runs_emulator_idx ON runs(emulator, subject);
CREATE INDEX runs_subject_idx ON runs(subject);
CREATE INDEX emulator_systems_system_idx ON emulator_systems(system);
CREATE INDEX checks_claim_idx ON checks(claim);
CREATE INDEX system_files_sha1_idx ON system_files(sha1);
"""
TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks",
"checks", "check_groups", "variants", "system_files", "benchmarks")
def _host_key(host: dict) -> tuple:
return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel",
"machine", "python"))
def write_sqlite(destination: Path, stored: list[mr.Stored], source: dict | None, profiles: dict,
scratch: Path) -> None:
"""A deterministic database of the records: sorted rows, no clock."""
scratch.mkdir(parents=True, exist_ok=True)
temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite"
temp.unlink(missing_ok=True)
destination.parent.mkdir(parents=True, exist_ok=True)
stored = sorted(stored, key=lambda s: (s.emulator, s.line))
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)}
connection = sqlite3.connect(temp)
try:
connection.executescript(SCHEMA_SQL)
meta = {
"schema_version": SQLITE_SCHEMA_VERSION,
"record_version": str(mr.RECORD_VERSION),
"measurements_repo": (source or {}).get("repo") or "",
"measurements_commit": (source or {}).get("commit") or "",
"license": "MIT",
"scope": "core probe records: the record lines, not their data files",
}
connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [
(sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")),
json_text(core.get("build_command")), json_text(core.get("compiler")),
json_text(core.get("library_name")), json_text(core.get("library_version")))
for sha, core in sorted(cores.items())])
connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [
(number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])])
emulators = sorted({s.emulator for s in stored})
connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [
(emu, system) for emu in emulators
for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))])
for entry in stored:
_insert_run(connection, entry, source or {}, hosts, entry.record["id"] not in live_ids)
connection.commit()
connection.execute("VACUUM")
finally:
connection.close()
os.replace(temp, destination)
def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
hosts: dict, superseded: bool) -> None:
r = entry.record
host, front, inputs = r["host"], r["frontend"], r["inputs"]
rid = r["id"]
connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
front.get("commit"), front["script_sha256"],
None if front.get("script_modified") is None else int(front["script_modified"]),
json_text(r["flags"]), r["data"]["sha256"], r["data"]["bytes"],
permalink(source.get("repo", ""), source.get("commit", ""), mr.records_path(entry.emulator), entry.line),
entry.line, r.get("migrated_from_version")))
content = inputs.get("content") or {}
connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, content.get("label"), content.get("sha1"), content.get("size"),
json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"],
inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]),
inputs["frame_hash_every"], inputs["joypad_ports"]))
connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [
(rid, position, track["file"], track["sha1"], track["size"])
for position, track in enumerate(content.get("tracks") or [])])
check = r.get("check")
if check:
result = check["result"]
connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, check["claim"], check["method"], check["observable"], check["variable"],
result["status"], result["statement"]))
connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
(rid, position, json_text(group["value"]) if "value" in group else None,
group.get("value_sha1"), json_text(group["variants"]))
for position, group in enumerate(result["groups"])])
for position, variant in enumerate(r["variants"]):
connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, position, variant["label"], variant["repetitions"], int(variant["loaded"]),
json_text(variant["flags"]), json_text(variant["observables"]),
json_text(variant["files_opened"]), json_text(variant["files_probed_absent"]),
json_text(variant["tracing"])))
connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
(rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
for metric, stats in sorted((r.get("timing") or {}).items()):
connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"]))
# --------------------------------------------------------------------------
# Building the site's files
# --------------------------------------------------------------------------
@dataclass
class Built:
index: dict
paths: list[Path]
def load_profiles(emulators_dir: Path) -> dict:
from common import load_emulator_profiles # noqa: PLC0415
return load_emulator_profiles(str(emulators_dir), skip_aliases=False)
def records_commit(root: Path) -> str | None:
"""The commit the record links name, with a warning when the records on
disk are not the ones it holds."""
commit = mr.last_commit(root)
if commit is None:
print(f"WARNING: no commit holds {root / mr.RECORDS_DIR}: the record links "
"lead nowhere until the records are committed.", file=sys.stderr)
return None
on_disk = {path: file.read_bytes() for path, file in mr.tree_files(root).items()}
if mr.committed_files(root, commit) != on_disk:
print(f"WARNING: {mr.RECORDS_DIR}/ differs from {commit[:12]}, the last commit that "
"touched it: links to records added since lead nowhere until they are "
"committed.", file=sys.stderr)
return commit
def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulators",
scratch: Path = ROOT / "tmp" / "site", system_names: dict[str, str] | None = None,
repo: str = REPO, commit: str | None = None) -> Built:
"""Write the index and the databases under the site's public folder.
``commit`` is the revision the record links name, by default the last
commit that touched the records.
"""
stored = mr.read_records(root)
source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)}
profiles = load_profiles(emulators_dir)
index = build_index(stored, source, profiles, system_names)
paths = [public / INDEX_PATH]
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
write_if_changed(str(public / INDEX_PATH),
json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
write_sqlite(public / SQLITE_PATH, stored, source, profiles, scratch)
paths.append(public / SQLITE_PATH)
for emulator in sorted({s.emulator for s in stored}):
target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite"
write_sqlite(target, [s for s in stored if s.emulator == emulator], source, profiles, scratch)
paths.append(target)
return Built(index, paths)
# --------------------------------------------------------------------------
# Pages
# --------------------------------------------------------------------------
def by_emulator(index: dict) -> dict[str, list[dict]]:
out: dict[str, list[dict]] = {}
for item in index.get("items", []):
@@ -91,7 +442,7 @@ def hosts_table(index: dict) -> list[str]:
computer = item["host"].split(", load ")[0]
hosts[computer] = hosts.get(computer, 0) + 1
if not hosts:
return ["No result is recorded at the pinned commit yet.", ""]
return ["No result is recorded yet.", ""]
return [
"| Computer | Tests |",
"|----------|------:|",
@@ -108,8 +459,8 @@ def with_hosts(page: str, index: dict) -> str:
def generate_explorer_page(index: dict) -> str:
commit = index.get("commit")
repo = index.get("repo") or ""
source = (f"[retrobios-measurements]({repo}/tree/{commit}) at {_short(commit)}"
if commit else "the measurements project, not read in this build")
source = (f"[`measurements/`]({repo}/tree/{commit}/{mr.RECORDS_DIR}) at {_short(commit)}"
if commit else f"`{mr.RECORDS_DIR}/` of this repository")
return "\n".join([
"# Measurements",
"",
@@ -148,8 +499,7 @@ def data_page_section(path: Path) -> list[str]:
"## Measurements",
"",
"[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe "
"record at the pinned commit of [retrobios-measurements]"
"(https://github.com/Abdess/retrobios-measurements), and "
f"record kept in [`{mr.RECORDS_DIR}/`]({REPO}/tree/main/{mr.RECORDS_DIR}), and "
"`downloads/measurements/<emulator>.sqlite` the records of one emulator, linked "
f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads "
"[`data/measurements/index.json`](data/measurements/index.json). A record line is "
@@ -160,7 +510,7 @@ def data_page_section(path: Path) -> list[str]:
"|-------|---------|",
*(f"| `{name}` | {', '.join(f'`{c}`' for c in columns)} |" for name, columns in sqlite_tables(path)),
"",
"`meta` names the measurements commit and the record version. A list or object "
"`meta` names the commit the record links point to and the record version. A list or object "
"value is JSON text, an absent one empty. `runs.superseded` is 1 for a record a "
"later one corrects.",
"",
scripts/sync_measurements.py
-564
View File
@@ -1,564 +0,0 @@
#!/usr/bin/env python3
"""Read the measurements repository at its pin and compile what the site shows.
The records live in their own repository (Abdess/retrobios-measurements),
one append-only .jsonl per emulator. This repository holds only the pin,
`measurements-pin.json`: the repository URL and the commit read. The clone
is kept at .cache/measurements/ and read at the pinned commit with git, so
its working tree may hold records not yet committed.
From the records at the pin it writes, never commits:
public/data/measurements/index.json the compact index the explorer and
the emulator pages read, one item
per emulator and subject
public/downloads/measurements.sqlite every record, normalised
public/downloads/measurements/<emu>.sqlite one database per emulator
`--fetch` clones or updates the clone, the only step that uses the network,
and refuses a remote whose history rewrote or removed a line the pin holds.
`--bump` then moves the pin to the fetched head.
"""
from __future__ import annotations
import argparse
import json
import os
import sqlite3
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import measurement_records as mr # noqa: E402
from artifacts import json_text, write_if_changed, write_text_atomic # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
PIN_FILE = ROOT / "measurements-pin.json"
CLONE = ROOT / ".cache" / "measurements"
INDEX_PATH = Path("data") / "measurements" / "index.json"
SQLITE_PATH = Path("downloads") / "measurements.sqlite"
PER_EMULATOR_DIR = Path("downloads") / "measurements"
SQLITE_SCHEMA_VERSION = "1"
INDEX_SCHEMA_VERSION = 1
class MeasurementsError(RuntimeError):
"""The measurements at the pin cannot be read, or break a rule."""
@dataclass(frozen=True)
class Stored:
"""One record line as read at a commit."""
emulator: str
line: int
record: dict
# --------------------------------------------------------------------------
# Pin and git
# --------------------------------------------------------------------------
def load_pin(path: Path = PIN_FILE) -> dict:
pin = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(pin.get("repo"), str) or not isinstance(pin.get("commit"), str) \
or len(pin["commit"]) != 40:
raise MeasurementsError(f"{path}: needs a repo URL and a 40-character commit")
return pin
def write_pin(pin: dict, path: Path = PIN_FILE) -> None:
write_text_atomic(str(path), json.dumps(pin, indent=2, sort_keys=True) + "\n")
def git(clone: Path, *args: str, binary: bool = False) -> str | bytes:
proc = subprocess.run(["git", "-C", str(clone), *args], capture_output=True, check=False)
if proc.returncode != 0:
raise MeasurementsError(f"git {' '.join(args)}: {proc.stderr.decode(errors='replace').strip()}")
return proc.stdout if binary else proc.stdout.decode("utf-8")
def has_commit(clone: Path, commit: str) -> bool:
if not (clone / ".git").exists():
return False
proc = subprocess.run(["git", "-C", str(clone), "cat-file", "-e", f"{commit}^{{commit}}"],
capture_output=True, check=False)
return proc.returncode == 0
def tree_files(clone: Path, commit: str) -> dict[str, str]:
"""Path -> blob id of every file under records/ and data/ at a commit."""
out: dict[str, str] = {}
listing = git(clone, "ls-tree", "-r", commit, "--", mr.RECORDS_DIR, mr.DATA_DIR)
for row in str(listing).splitlines():
meta, _, path = row.partition("\t")
out[path] = meta.split()[2]
return out
def show(clone: Path, commit: str, path: str) -> bytes:
return bytes(git(clone, "show", f"{commit}:{path}", binary=True))
# --------------------------------------------------------------------------
# Reading and checking
# --------------------------------------------------------------------------
def read_records(clone: Path, commit: str) -> list[Stored]:
"""Every record line at a commit, each checked against its rules and data."""
files = tree_files(clone, commit)
problems: list[str] = []
stored: list[Stored] = []
for path in sorted(p for p in files if p.startswith(f"{mr.RECORDS_DIR}/")):
emulator = path[len(mr.RECORDS_DIR) + 1:].removesuffix(".jsonl")
if not path.endswith(".jsonl") or "/" in emulator:
problems.append(f"{path}: records/ holds <emulator>.jsonl files only")
continue
seen: set[str] = set()
try:
records = mr.parse_lines(show(clone, commit, path).decode("utf-8"), path)
except mr.RecordError as exc:
problems.append(str(exc))
continue
for number, record in enumerate(records, 1):
where = f"{path}:{number}"
problems.extend(f"{where}: {p}" for p in mr.check_line(record))
if record.get("emulator") != emulator:
problems.append(f"{where}: emulator {record.get('emulator')!r} in {path}")
if record.get("id") in seen:
problems.append(f"{where}: id {record.get('id')} appears twice")
if record.get("supersedes") and record["supersedes"] not in seen:
problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file")
seen.add(record.get("id"))
data = mr.sidecar_path(emulator, str(record.get("id")))
if data not in files:
problems.append(f"{where}: {data} is missing")
else:
problems.extend(f"{where}: {p}" for p in mr.check_pair(record, show(clone, commit, data)))
stored.append(Stored(emulator, number, record))
if problems:
raise MeasurementsError("records at {}:\n {}".format(commit[:12], "\n ".join(problems)))
return stored
def append_only_problems(clone: Path, old: str, new: str) -> list[str]:
"""What ``new`` changed or removed of the lines and data ``old`` holds."""
before, after = tree_files(clone, old), tree_files(clone, new)
problems = []
for path, blob in sorted(before.items()):
if path not in after:
problems.append(f"{path}: removed")
elif after[path] == blob:
continue
elif path.endswith(".jsonl"):
moved = mr.appended_only(show(clone, old, path).decode("utf-8"),
show(clone, new, path).decode("utf-8"))
if moved:
problems.append(f"{path}: {moved}")
else:
problems.append(f"{path}: rewritten")
return problems
# --------------------------------------------------------------------------
# Index
# --------------------------------------------------------------------------
def host_summary(host: dict) -> str:
parts = [host.get("cpu_model") or "unknown CPU"]
if host.get("cpu_count"):
parts.append(f"{host['cpu_count']} CPUs")
parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip()
if "Linux" in host.get("os", "") else host.get("os", ""))
parts.append(f"Python {host.get('python', '')}")
before, after = host.get("load_before"), host.get("load_after")
if before and after:
parts.append(f"load {before[0]} before, {after[0]} after")
return ", ".join(part for part in parts if part)
def bench_statement(record: dict) -> str:
timing = record["timing"]
frame, fps = timing["ms_per_frame"], timing["fps"]
return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), "
f"min {frame['min']:.3f}, max {frame['max']:.3f}, over "
f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames")
def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str:
return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "")
def index_item(entry: Stored, runs: int, pin: dict, profiles: dict,
system_names: dict[str, str]) -> dict:
record = entry.record
profile = profiles.get(entry.emulator) or {}
check = record.get("check") or {}
core = record["core"]
content = (record["inputs"].get("content") or {})
return {
"emulator": entry.emulator,
"emulator_name": str(profile.get("emulator", entry.emulator)),
"profiled": bool(profile),
"systems": sorted(set(profile.get("systems") or [])),
"system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))],
"subject": record["subject"],
"kind": record["kind"],
"claim": check.get("claim"),
"method": check.get("method"),
"observable": check.get("observable"),
"variable": check.get("variable"),
"status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"),
"statement": (check.get("result") or {}).get("statement")
or (bench_statement(record) if record.get("timing") else ""),
"content": content.get("label"),
"variants": len(record["variants"]),
"repetitions": record["inputs"]["repetitions"],
"flags": len(record["flags"]),
"date": record["recorded_at"],
"host": host_summary(record["host"]),
"core": {
"library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])),
"file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"),
"commit": core.get("commit"), "build": core.get("build_command"),
},
"frontend_commit": record["frontend"].get("commit"),
"id": record["id"],
"runs": runs,
"record_url": permalink(pin["repo"], pin["commit"], mr.records_path(entry.emulator), entry.line),
"data_url": permalink(pin["repo"], pin["commit"], mr.sidecar_path(entry.emulator, record["id"])),
"sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
}
def build_index(stored: list[Stored], pin: dict | None, profiles: dict,
system_names: dict[str, str] | None = None) -> dict:
"""One item per (emulator, subject): its latest record not superseded."""
by_subject: dict[tuple[str, str], list[Stored]] = {}
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
for entry in stored:
if entry.record["id"] in live_ids:
by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry)
items = []
for key in sorted(by_subject):
entries = by_subject[key]
latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line))
items.append(index_item(latest, len(entries), pin or {}, profiles, system_names or {}))
items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"]))
return {
"schema_version": INDEX_SCHEMA_VERSION,
"kind": "measurements",
"repo": (pin or {}).get("repo"),
"commit": (pin or {}).get("commit"),
"records": len(stored),
"count": len(items),
"items": items,
}
# --------------------------------------------------------------------------
# SQLite
# --------------------------------------------------------------------------
SCHEMA_SQL = """
PRAGMA journal_mode = OFF;
PRAGMA synchronous = OFF;
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE cores (
sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT,
build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT
);
CREATE TABLE hosts (
id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL,
kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL
);
CREATE TABLE emulator_systems (
emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system)
);
CREATE TABLE runs (
id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL,
kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT,
superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
script_modified INTEGER, flags TEXT NOT NULL, data_sha256 TEXT NOT NULL,
data_bytes INTEGER NOT NULL, record_url TEXT NOT NULL, line INTEGER NOT NULL,
migrated_from_version INTEGER
);
CREATE TABLE inputs (
run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT,
content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL,
warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL,
observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL
);
CREATE TABLE content_tracks (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL,
PRIMARY KEY (run_id, position)
);
CREATE TABLE checks (
run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL,
observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL,
statement TEXT NOT NULL
);
CREATE TABLE check_groups (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position)
);
CREATE TABLE variants (
run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL,
label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL,
flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT,
files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position)
);
CREATE TABLE system_files (
run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL,
destination TEXT NOT NULL, sha1 TEXT,
PRIMARY KEY (run_id, variant_position, destination)
);
CREATE TABLE benchmarks (
run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL,
median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL,
min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric)
);
CREATE INDEX runs_emulator_idx ON runs(emulator, subject);
CREATE INDEX runs_subject_idx ON runs(subject);
CREATE INDEX emulator_systems_system_idx ON emulator_systems(system);
CREATE INDEX checks_claim_idx ON checks(claim);
CREATE INDEX system_files_sha1_idx ON system_files(sha1);
"""
TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks",
"checks", "check_groups", "variants", "system_files", "benchmarks")
def _host_key(host: dict) -> tuple:
return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel",
"machine", "python"))
def write_sqlite(destination: Path, stored: list[Stored], pin: dict | None, profiles: dict,
scratch: Path) -> None:
"""A deterministic database of the records: sorted rows, no clock."""
scratch.mkdir(parents=True, exist_ok=True)
temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite"
temp.unlink(missing_ok=True)
destination.parent.mkdir(parents=True, exist_ok=True)
stored = sorted(stored, key=lambda s: (s.emulator, s.line))
live_ids = {r["id"] for r in mr.live(s.record for s in stored)}
hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)}
connection = sqlite3.connect(temp)
try:
connection.executescript(SCHEMA_SQL)
meta = {
"schema_version": SQLITE_SCHEMA_VERSION,
"record_version": str(mr.RECORD_VERSION),
"measurements_repo": (pin or {}).get("repo") or "",
"measurements_commit": (pin or {}).get("commit") or "",
"license": "MIT",
"scope": "core probe records: the record lines, not their data files",
}
connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [
(sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")),
json_text(core.get("build_command")), json_text(core.get("compiler")),
json_text(core.get("library_name")), json_text(core.get("library_version")))
for sha, core in sorted(cores.items())])
connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [
(number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])])
emulators = sorted({s.emulator for s in stored})
connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [
(emu, system) for emu in emulators
for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))])
for entry in stored:
_insert_run(connection, entry, pin or {}, hosts, entry.record["id"] not in live_ids)
connection.commit()
connection.execute("VACUUM")
finally:
connection.close()
os.replace(temp, destination)
def _insert_run(connection: sqlite3.Connection, entry: Stored, pin: dict,
hosts: dict, superseded: bool) -> None:
r = entry.record
host, front, inputs = r["host"], r["frontend"], r["inputs"]
rid = r["id"]
connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
front.get("commit"), front["script_sha256"],
None if front.get("script_modified") is None else int(front["script_modified"]),
json_text(r["flags"]), r["data"]["sha256"], r["data"]["bytes"],
permalink(pin.get("repo", ""), pin.get("commit", ""), mr.records_path(entry.emulator), entry.line),
entry.line, r.get("migrated_from_version")))
content = inputs.get("content") or {}
connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, content.get("label"), content.get("sha1"), content.get("size"),
json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"],
inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]),
inputs["frame_hash_every"], inputs["joypad_ports"]))
connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [
(rid, position, track["file"], track["sha1"], track["size"])
for position, track in enumerate(content.get("tracks") or [])])
check = r.get("check")
if check:
result = check["result"]
connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, check["claim"], check["method"], check["observable"], check["variable"],
result["status"], result["statement"]))
connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
(rid, position, json_text(group["value"]) if "value" in group else None,
group.get("value_sha1"), json_text(group["variants"]))
for position, group in enumerate(result["groups"])])
for position, variant in enumerate(r["variants"]):
connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
rid, position, variant["label"], variant["repetitions"], int(variant["loaded"]),
json_text(variant["flags"]), json_text(variant["observables"]),
json_text(variant["files_opened"]), json_text(variant["files_probed_absent"]),
json_text(variant["tracing"])))
connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
(rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
for metric, stats in sorted((r.get("timing") or {}).items()):
connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", (
rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"]))
# --------------------------------------------------------------------------
# Building the site's files
# --------------------------------------------------------------------------
@dataclass
class Built:
index: dict
paths: list[Path]
def load_profiles(emulators_dir: Path) -> dict:
from common import load_emulator_profiles # noqa: PLC0415
return load_emulator_profiles(str(emulators_dir), skip_aliases=False)
def records_at_pin(clone: Path, pin: dict, require: bool) -> list[Stored] | None:
"""The records at the pin, or None when the clone does not hold it."""
if has_commit(clone, pin["commit"]):
return read_records(clone, pin["commit"])
message = (f"measurements commit {pin['commit'][:12]} is not in {clone}: run "
f"python scripts/sync_measurements.py --fetch")
if require:
raise MeasurementsError(message)
print(f"WARNING: {message}. The site lists no measurements.", file=sys.stderr)
return None
def build(public: Path, clone: Path = CLONE, pin_path: Path = PIN_FILE,
emulators_dir: Path = ROOT / "emulators", require: bool = False,
scratch: Path = ROOT / "tmp" / "site",
system_names: dict[str, str] | None = None) -> Built:
"""Write the index and the databases under the site's public folder."""
pin = load_pin(pin_path)
stored = records_at_pin(clone, pin, require)
profiles = load_profiles(emulators_dir)
if stored is None:
pin = None
index = build_index(stored or [], pin, profiles, system_names)
paths = [public / INDEX_PATH]
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
write_if_changed(str(public / INDEX_PATH),
json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n")
write_sqlite(public / SQLITE_PATH, stored or [], pin, profiles, scratch)
paths.append(public / SQLITE_PATH)
for emulator in sorted({s.emulator for s in stored or []}):
target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite"
write_sqlite(target, [s for s in stored or [] if s.emulator == emulator], pin, profiles, scratch)
paths.append(target)
return Built(index, paths)
def fetch(clone: Path, pin: dict) -> str:
"""Clone or update the measurements repository. Returns the remote head."""
if not (clone / ".git").exists():
clone.parent.mkdir(parents=True, exist_ok=True)
proc = subprocess.run(["git", "clone", "--quiet", pin["repo"], str(clone)],
capture_output=True, text=True, check=False)
if proc.returncode != 0:
raise MeasurementsError(f"git clone {pin['repo']}: {proc.stderr.strip()}")
else:
git(clone, "fetch", "--quiet", "origin")
head = str(git(clone, "rev-parse", "refs/remotes/origin/HEAD")).strip()
branch = str(git(clone, "rev-parse", "--abbrev-ref", "HEAD")).strip()
if branch != "HEAD":
proc = subprocess.run(["git", "-C", str(clone), "merge", "--ff-only", "--quiet", head],
capture_output=True, text=True, check=False)
if proc.returncode != 0:
print(f"WARNING: {clone} has local commits, its branch was left as is: "
f"{proc.stderr.strip()}", file=sys.stderr)
return head
def remote_head(repo: str) -> str:
proc = subprocess.run(["git", "ls-remote", repo, "HEAD"], capture_output=True, text=True,
check=False, timeout=60)
if proc.returncode != 0 or not proc.stdout.strip():
raise MeasurementsError(f"git ls-remote {repo}: {proc.stderr.strip() or 'no HEAD'}")
return proc.stdout.split()[0]
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0])
parser.add_argument("--fetch", action="store_true",
help="clone or update the clone, and check the remote kept every line the pin holds")
parser.add_argument("--bump", action="store_true",
help=f"after --fetch, move {PIN_FILE.name} to the fetched head")
parser.add_argument("--check-only", action="store_true",
help="check the records at the pin and write nothing")
parser.add_argument("--clone", default=str(CLONE), help="the clone (default: %(default)s)")
parser.add_argument("--pin", default=str(PIN_FILE), help="the pin file (default: %(default)s)")
parser.add_argument("--site-dir", default=str(ROOT / "website"),
help="the site project whose public/ folder is written (default: %(default)s)")
args = parser.parse_args(argv)
if args.bump and not args.fetch:
parser.error("--bump needs --fetch")
clone, pin_path = Path(args.clone), Path(args.pin)
try:
pin = load_pin(pin_path)
if args.fetch:
head = fetch(clone, pin)
problems = append_only_problems(clone, pin["commit"], head) \
if has_commit(clone, pin["commit"]) else [f"pin {pin['commit']} is not in the remote history"]
if problems:
print("ERROR: the remote changed records the pin holds:\n " + "\n ".join(problems),
file=sys.stderr)
return 1
records = read_records(clone, head)
pinned = len(read_records(clone, pin["commit"]))
print(f"{head[:12]}: {len(records)} records, {len(records) - pinned} after the pin "
f"{pin['commit'][:12]}")
if args.bump and head != pin["commit"]:
write_pin({**pin, "commit": head}, pin_path)
print(f"pin moved to {head}")
return 0
if args.check_only:
stored = records_at_pin(clone, pin, require=True)
print(f"{pin['commit'][:12]}: {len(stored or [])} records, every line and data file checked")
return 0
built = build(Path(args.site_dir) / "public", clone, pin_path, require=True)
print(f"{built.index['records']} records, {built.index['count']} subjects: "
+ ", ".join(str(p.relative_to(Path(args.site_dir))) for p in built.paths))
return 0
except MeasurementsError as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1
if __name__ == "__main__":
from common import run_cli
run_cli(main)
scripts/validate_schemas.py
+14 -27
View File
@@ -593,46 +593,33 @@ def _format_errors() -> list[str]:
]
def _measurement_errors(clone: Path | None = None, pin_path: Path | None = None) -> list[str]:
"""Every record line and data file at the measurements pin, against their
schemas and the rules sync_measurements.py applies.
The records live in their own repository. Without a clone holding the
pin there is nothing to read, and the run says so.
"""
import sync_measurements # noqa: PLC0415
def _measurement_errors(root: Path | None = None) -> list[str]:
"""Every record line and data file under measurements/, against their
schemas and the rules measurement_records.read_records applies."""
import measurement_records as mr # noqa: PLC0415
clone = clone or sync_measurements.CLONE
pin = sync_measurements.load_pin(pin_path or sync_measurements.PIN_FILE)
if not sync_measurements.has_commit(clone, pin["commit"]):
print(f"SKIP measurements: {pin['commit'][:12]} is not in {clone} "
"(python scripts/sync_measurements.py --fetch)")
return []
commit = pin["commit"]
root = root or ROOT
record_validator = _validator("core-probe-record.schema.json")
data_validator = _validator("core-probe-data.schema.json")
out: list[str] = []
files = sync_measurements.tree_files(clone, commit)
files = mr.tree_files(root)
for path in sorted(p for p in files if p.endswith(".jsonl")):
text = sync_measurements.show(clone, commit, path).decode("utf-8")
for number, line in enumerate(text.splitlines(), 1):
emulator = path[len(mr.RECORDS_DIR) + 1:].removesuffix(".jsonl")
for number, line in enumerate(files[path].read_text(encoding="utf-8").splitlines(), 1):
try:
record = json.loads(line)
except json.JSONDecodeError as exc:
out.append(f"measurements {path}:{number}: {exc}")
out.append(f"{path}:{number}: {exc}")
continue
out.extend(_errors(record_validator, record, f"measurements {path}:{number}"))
data = mr.sidecar_path(path[len("records/"):-len(".jsonl")], str(record.get("id")))
out.extend(_errors(record_validator, record, f"{path}:{number}"))
data = mr.sidecar_path(emulator, str(record.get("id")))
if data in files:
out.extend(_errors(data_validator,
mr.read_sidecar(sync_measurements.show(clone, commit, data)),
f"measurements {data}"))
out.extend(_errors(data_validator, mr.read_sidecar(files[data].read_bytes()), data))
if not out:
try:
sync_measurements.read_records(clone, commit)
except sync_measurements.MeasurementsError as exc:
out.append(f"measurements {exc}")
mr.read_records(root)
except mr.MeasurementsError as exc:
out.append(str(exc))
return out
tests/test_core_probe.py
+11 -15
View File
@@ -263,9 +263,10 @@ class FakeCoreTests(unittest.TestCase):
self.assertEqual(record["emulator"], "fake")
self.assertEqual(sorted(p.relative_to(record_dir).as_posix()
for p in record_dir.rglob("*") if p.is_file()),
sorted(["records/fake.jsonl", f"data/fake/{self._only_id(record_dir)}.json.gz"]))
line = json.loads((record_dir / "records" / "fake.jsonl").read_text())
blob = (record_dir / "data" / "fake" / f"{line['id']}.json.gz").read_bytes()
sorted(["measurements/fake.jsonl",
f"measurements/data/fake/{self._only_id(record_dir)}.json.gz"]))
line = json.loads((record_dir / "measurements" / "fake.jsonl").read_text())
blob = (record_dir / "measurements" / "data" / "fake" / f"{line['id']}.json.gz").read_bytes()
self.assertEqual(measurement_records.check_line(line), [])
self.assertEqual(measurement_records.check_pair(line, blob), [])
# The line and its sidecar give back the record the run printed.
@@ -281,14 +282,14 @@ class FakeCoreTests(unittest.TestCase):
@staticmethod
def _only_id(record_dir: Path) -> str:
return json.loads((record_dir / "records" / "fake.jsonl").read_text())["id"]
return json.loads((record_dir / "measurements" / "fake.jsonl").read_text())["id"]
def test_second_run_is_appended_beside_the_first(self) -> None:
record_dir = self.tmp / "appended"
argv = ["run", "--core", str(self.core), "--frames", "1", "--no-trace", "--max-load", "1000",
"--record", str(record_dir), "--subject", "smoke"]
self.assertEqual(self.probe(argv)[0], 0)
lines = record_dir / "records" / "fake.jsonl"
lines = record_dir / "measurements" / "fake.jsonl"
first = lines.read_text()
code, record = self.probe(argv)
self.assertEqual(code, 0)
@@ -301,16 +302,11 @@ class FakeCoreTests(unittest.TestCase):
self.assertEqual(code, 2)
self.assertEqual(len(lines.read_text().splitlines()), 3)
def test_recording_inside_the_tracked_tree_is_refused(self) -> None:
inside = REPO_ROOT / "measurements"
err = io.StringIO()
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
code = core_probe.main(["run", "--core", str(self.core), "--frames", "1", "--no-trace",
"--record", str(inside), "--work-dir", str(self.tmp / "work")])
self.assertEqual(code, 2)
self.assertIn("inside the tracked tree", err.getvalue())
self.assertIn(".cache/measurements", err.getvalue())
self.assertFalse(inside.exists())
def test_record_defaults_to_this_checkout(self) -> None:
parser = core_probe.build_parser()
args = parser.parse_args(["run", "--core", sys.executable, "--record", "--frames", "1"])
self.assertEqual(Path(args.record), REPO_ROOT)
self.assertIsNone(parser.parse_args(["run", "--core", sys.executable]).record)
def test_bench_stats(self) -> None:
code, record = self.probe(["bench", "--core", str(self.core), "--frames", "2", "--warmup", "1",
tests/test_measurements.py
+160 -117
View File
@@ -1,16 +1,15 @@
"""Core probe records in their own repository, and what the site makes of them.
"""Core probe records under measurements/, and what the site makes of them.
A synthetic measurements repository is built with git under tmp/tests:
records split into a line and a sidecar, read at a pinned commit, compiled
into the index and the SQLite files. The append-only rule is checked the
way --fetch applies it, and two branches that each add a line merge.
Synthetic record trees are built under tmp/tests: records split into a line
and a sidecar, read from the tree, compiled into the index and the SQLite
files. The append-only rule is checked against a commit, two branches that
each add a line merge, and the records this repository holds are checked
against HEAD.
"""
from __future__ import annotations
import contextlib
import copy
import io
import json
import os
import re
@@ -26,10 +25,10 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(REPO_ROOT / "scripts"))
import measurement_records as mr # noqa: E402
import sitemeasurements # noqa: E402
import sync_measurements as sm # noqa: E402
import sitemeasurements as sm # noqa: E402
REPO_URL = "https://github.com/Abdess/retrobios-measurements"
REPO_URL = "https://github.com/Abdess/retrobios"
COMMIT = "e" * 40
SHA1 = "a" * 40
SHA256 = "b" * 64
@@ -95,6 +94,8 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check",
return record
def git(cwd: Path, *args: str) -> str:
env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t",
"GIT_COMMITTER_EMAIL": "t@t", "GIT_CEILING_DIRECTORIES": str(cwd.parent)}
@@ -122,32 +123,33 @@ def commit_all(tree: Path, message: str) -> str:
class Workspace(unittest.TestCase):
"""A measurements repository with three records and a pin on it."""
"""A tree with three records under measurements/ and one profile."""
def setUp(self) -> None:
(REPO_ROOT / "tmp" / "tests").mkdir(parents=True, exist_ok=True)
self.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests"))
self.addCleanup(shutil.rmtree, self.tmp, True)
self.repo = self.tmp / "remote"
self.repo.mkdir()
git(self.repo, "init", "-q", "-b", "main")
(self.repo / ".gitattributes").write_text("records/*.jsonl merge=union\n")
self.first = add_record(self.repo, full_record("alpha", "card", "2026-10-01T10:00:00+00:00"))
self.bench = add_record(self.repo, full_record("alpha", "speed", "2026-10-01T11:00:00+00:00",
self.tree = self.tmp / "tree"
self.tree.mkdir()
self.first = add_record(self.tree, full_record("alpha", "card", "2026-10-01T10:00:00+00:00"))
self.bench = add_record(self.tree, full_record("alpha", "speed", "2026-10-01T11:00:00+00:00",
kind="bench"))
self.beta = add_record(self.repo, full_record("beta", "card", "2026-10-02T10:00:00+00:00",
self.beta = add_record(self.tree, full_record("beta", "card", "2026-10-02T10:00:00+00:00",
values=("1" * 40, "1" * 40)))
self.commit = commit_all(self.repo, "feat: first records")
self.pin = self.tmp / "pin.json"
sm.write_pin({"repo": str(self.repo), "commit": self.commit}, self.pin)
self.emulators = self.tmp / "emulators"
self.emulators.mkdir()
(self.emulators / "alpha.yml").write_text(
'emulator: "Alpha"\ntype: libretro\nsystems:\n - sega-saturn\nfiles: []\n')
def built(self, public: Path, clone: Path | None = None) -> sm.Built:
return sm.build(public, clone or self.repo, self.pin, self.emulators, require=True,
scratch=self.tmp / "scratch", system_names={"sega-saturn": "Sega Saturn"})
def built(self, public: Path) -> sm.Built:
return sm.build(public, self.tree, self.emulators, scratch=self.tmp / "scratch",
system_names={"sega-saturn": "Sega Saturn"}, repo=REPO_URL, commit=COMMIT)
def repository(self) -> str:
"""Make the tree a git repository, its records committed on main."""
git(self.tree, "init", "-q", "-b", "main")
(self.tree / ".gitattributes").write_text("measurements/*.jsonl merge=union\n")
return commit_all(self.tree, "feat: first records")
class RecordSplit(unittest.TestCase):
@@ -211,17 +213,29 @@ class RecordSplit(unittest.TestCase):
self.assertTrue(mr.check_pair(forged, blob))
class SyncAndIndex(Workspace):
def test_records_are_read_at_the_pin_not_the_working_tree(self) -> None:
add_record(self.repo, full_record("alpha", "card", "2026-10-03T10:00:00+00:00"))
records = sm.read_records(self.repo, self.commit)
self.assertEqual(len(records), 3)
class ReadAndIndex(Workspace):
def test_records_are_read_from_the_tree(self) -> None:
records = mr.read_records(self.tree)
self.assertEqual([(s.emulator, s.line) for s in records], [("alpha", 1), ("alpha", 2), ("beta", 1)])
self.assertEqual(mr.read_records(self.tmp / "empty"), [])
def test_a_broken_tree_is_named(self) -> None:
stray = self.tree / mr.DATA_DIR / "beta" / "2026-10-02-000000000000.json.gz"
stray.write_bytes(b"x")
(self.tree / mr.sidecar_path("alpha", self.bench["id"])).unlink()
with self.assertRaises(mr.MeasurementsError) as caught:
mr.read_records(self.tree)
message = str(caught.exception)
self.assertIn(f"{mr.sidecar_path('alpha', self.bench['id'])} is missing", message)
self.assertIn(f"{mr.DATA_DIR}/beta/2026-10-02-000000000000.json.gz: no record line names it", message)
def test_index_shape(self) -> None:
index = self.built(self.tmp / "public").index
self.assertEqual({k: index[k] for k in ("schema_version", "kind", "repo", "commit", "records", "count")},
{"schema_version": 1, "kind": "measurements", "repo": str(self.repo),
"commit": self.commit, "records": 3, "count": 3})
{"schema_version": 1, "kind": "measurements", "repo": REPO_URL,
"commit": COMMIT, "records": 3, "count": 3})
keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind",
"claim", "method", "observable", "variable", "status", "statement", "content",
"variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id",
@@ -232,18 +246,28 @@ class SyncAndIndex(Workspace):
self.assertEqual(alpha["card"]["status"], "differs")
self.assertEqual(alpha["card"]["system_names"], ["Sega Saturn"])
self.assertEqual(alpha["card"]["record_url"],
f"{self.repo}/blob/{self.commit}/records/alpha.jsonl#L1")
f"{REPO_URL}/blob/{COMMIT}/measurements/alpha.jsonl#L1")
self.assertEqual(alpha["card"]["data_url"],
f"{REPO_URL}/blob/{COMMIT}/measurements/data/alpha/{self.first['id']}.json.gz")
self.assertEqual(alpha["speed"]["status"], "measured")
self.assertIn("ms per frame", alpha["speed"]["statement"])
self.assertIn("Test CPU", alpha["speed"]["host"])
written = json.loads((self.tmp / "public" / sm.INDEX_PATH).read_text())
self.assertEqual(written, index)
def test_links_name_the_last_commit_that_touched_the_records(self) -> None:
first = self.repository()
(self.tree / "other.txt").write_text("x")
commit_all(self.tree, "chore: unrelated")
self.assertEqual(mr.last_commit(self.tree), first)
index = sm.build(self.tmp / "public", self.tree, self.emulators, scratch=self.tmp / "scratch").index
self.assertEqual(index["commit"], first)
self.assertTrue(all(f"/blob/{first}/measurements/" in i["record_url"] for i in index["items"]))
def test_latest_record_of_a_subject_wins_and_a_superseded_one_is_dropped(self) -> None:
later = full_record("alpha", "card", "2026-10-05T10:00:00+00:00", values=("3" * 40, "3" * 40))
later["supersedes"] = self.first["id"]
add_record(self.repo, later)
sm.write_pin({"repo": str(self.repo), "commit": commit_all(self.repo, "feat: rerun")}, self.pin)
add_record(self.tree, later)
card = next(i for i in self.built(self.tmp / "public").index["items"]
if i["emulator"] == "alpha" and i["subject"] == "card")
self.assertEqual((card["status"], card["runs"]), ("same", 1))
@@ -257,7 +281,7 @@ class SyncAndIndex(Workspace):
"SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")]
self.assertEqual(tuple(tables), sm.TABLES)
count = lambda table: connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: E731
records = [s.record for s in sm.read_records(self.repo, self.commit)]
records = [s.record for s in mr.read_records(self.tree)]
self.assertEqual(count("runs"), len(records))
self.assertEqual(count("checks"), sum(1 for r in records if r.get("check")))
self.assertEqual(count("benchmarks"), 2 * sum(1 for r in records if r.get("timing")))
@@ -266,10 +290,12 @@ class SyncAndIndex(Workspace):
sum(len(r["check"]["result"]["groups"]) for r in records if r.get("check")))
self.assertEqual(count("hosts"), 1)
self.assertEqual(count("cores"), 1)
self.assertEqual(dict(connection.execute("SELECT key, value FROM meta"))["measurements_commit"],
self.commit)
meta = dict(connection.execute("SELECT key, value FROM meta"))
self.assertEqual((meta["measurements_repo"], meta["measurements_commit"]), (REPO_URL, COMMIT))
self.assertEqual(connection.execute("SELECT system FROM emulator_systems").fetchall(),
[("sega-saturn",)])
self.assertEqual(connection.execute("SELECT record_url FROM runs WHERE line = 1 AND emulator = 'beta'")
.fetchone()[0], f"{REPO_URL}/blob/{COMMIT}/measurements/beta.jsonl#L1")
indexes = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'index'")}
self.assertTrue({"runs_emulator_idx", "emulator_systems_system_idx", "checks_claim_idx"} <= indexes)
connection.close()
@@ -286,15 +312,11 @@ class SyncAndIndex(Workspace):
for path in first.paths:
self.assertEqual(path.read_bytes(), (two / path.relative_to(one)).read_bytes(), path.name)
def test_a_missing_pin_fails_only_when_required(self) -> None:
sm.write_pin({"repo": str(self.repo), "commit": "f" * 40}, self.pin)
with self.assertRaises(sm.MeasurementsError):
self.built(self.tmp / "public")
with contextlib.redirect_stderr(io.StringIO()) as err:
built = sm.build(self.tmp / "public", self.repo, self.pin, self.emulators, require=False,
scratch=self.tmp / "scratch")
self.assertIn("WARNING", err.getvalue())
self.assertEqual((built.index["count"], built.index["commit"]), (0, None))
def test_no_records_writes_an_empty_index(self) -> None:
shutil.rmtree(self.tree / mr.RECORDS_DIR)
built = self.built(self.tmp / "public")
self.assertEqual((built.index["records"], built.index["count"]), (0, 0))
self.assertEqual(len(built.paths), 2)
def test_every_line_validates_against_the_schema(self) -> None:
try:
@@ -303,68 +325,82 @@ class SyncAndIndex(Workspace):
self.skipTest("jsonschema is not installed")
import validate_schemas # noqa: PLC0415
self.assertEqual(validate_schemas._measurement_errors(self.repo, self.pin), [])
lines = self.repo / mr.records_path("beta")
self.assertEqual(validate_schemas._measurement_errors(self.tree), [])
lines = self.tree / mr.records_path("beta")
record = json.loads(lines.read_text())
record["kind"] = "guess"
lines.write_text(mr.line_text(record) + "\n")
sm.write_pin({"repo": str(self.repo), "commit": commit_all(self.repo, "bad")}, self.pin)
errors = validate_schemas._measurement_errors(self.repo, self.pin)
self.assertTrue(any("records/beta.jsonl:1" in e and "kind" in e for e in errors), errors)
errors = validate_schemas._measurement_errors(self.tree)
self.assertTrue(any("measurements/beta.jsonl:1" in e and "kind" in e for e in errors), errors)
self.assertIsNotNone(Draft202012Validator)
class AppendOnly(Workspace):
def fetch(self, *extra: str) -> tuple[int, str]:
err = io.StringIO()
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err):
code = sm.main(["--fetch", "--clone", str(self.tmp / "clone"), "--pin", str(self.pin), *extra])
return code, err.getvalue()
def setUp(self) -> None:
super().setUp()
self.commit = self.repository()
def test_an_appended_line_moves_the_pin(self) -> None:
self.assertEqual(self.fetch()[0], 0)
add_record(self.repo, full_record("alpha", "card", "2026-10-04T10:00:00+00:00"))
head = commit_all(self.repo, "feat: rerun")
self.assertEqual(self.fetch("--bump")[0], 0)
self.assertEqual(sm.load_pin(self.pin)["commit"], head)
def test_an_appended_line_and_a_new_file_pass(self) -> None:
add_record(self.tree, full_record("alpha", "card", "2026-10-04T10:00:00+00:00"))
add_record(self.tree, full_record("gamma", "card", "2026-10-04T10:00:00+00:00"))
self.assertEqual(mr.append_only_problems(self.tree, "HEAD"), [])
def test_a_changed_line_is_refused(self) -> None:
lines = self.repo / mr.records_path("alpha")
def test_a_changed_line_is_named(self) -> None:
lines = self.tree / mr.records_path("alpha")
text = lines.read_text().splitlines()
text[0] = text[0].replace('"Each card, twice."', '"Each card, once."')
lines.write_text("\n".join(text) + "\n")
commit_all(self.repo, "rewrite")
code, err = self.fetch("--bump")
self.assertEqual(code, 1)
self.assertIn("records/alpha.jsonl: line 1 changed", err)
self.assertEqual(sm.load_pin(self.pin)["commit"], self.commit)
self.assertEqual(mr.append_only_problems(self.tree, "HEAD"),
["measurements/alpha.jsonl: line 1 changed"])
def test_a_removed_line_or_data_file_is_refused(self) -> None:
lines = self.repo / mr.records_path("alpha")
def test_a_removed_line_or_data_file_is_named(self) -> None:
lines = self.tree / mr.records_path("alpha")
lines.write_text(lines.read_text().splitlines()[0] + "\n")
(self.repo / mr.sidecar_path("beta", self.beta["id"])).unlink()
commit_all(self.repo, "remove")
code, err = self.fetch()
self.assertEqual(code, 1)
self.assertIn("records/alpha.jsonl: line 2 removed", err)
self.assertIn(f"{mr.sidecar_path('beta', self.beta['id'])}: removed", err)
(self.tree / mr.sidecar_path("beta", self.beta["id"])).unlink()
problems = mr.append_only_problems(self.tree, "HEAD")
self.assertIn("measurements/alpha.jsonl: line 2 removed", problems)
self.assertIn(f"{mr.sidecar_path('beta', self.beta['id'])}: removed", problems)
def test_a_rewritten_data_file_is_named(self) -> None:
data = self.tree / mr.sidecar_path("beta", self.beta["id"])
data.write_bytes(data.read_bytes() + b"x")
self.assertEqual(mr.append_only_problems(self.tree, "HEAD"),
[f"{mr.sidecar_path('beta', self.beta['id'])}: rewritten"])
def test_two_branches_adding_a_line_merge_cleanly(self) -> None:
git(self.repo, "checkout", "-q", "-b", "left")
add_record(self.repo, full_record("alpha", "card", "2026-10-06T10:00:00+00:00"))
commit_all(self.repo, "left")
git(self.repo, "checkout", "-q", "main")
git(self.repo, "checkout", "-q", "-b", "right")
add_record(self.repo, full_record("alpha", "speed", "2026-10-06T11:00:00+00:00", kind="bench",
git(self.tree, "checkout", "-q", "-b", "left")
add_record(self.tree, full_record("alpha", "card", "2026-10-06T10:00:00+00:00"))
commit_all(self.tree, "left")
git(self.tree, "checkout", "-q", "main")
git(self.tree, "checkout", "-q", "-b", "right")
add_record(self.tree, full_record("alpha", "speed", "2026-10-06T11:00:00+00:00", kind="bench",
values=("4" * 40,)))
commit_all(self.repo, "right")
git(self.repo, "merge", "-q", "--no-edit", "left")
merged = git(self.repo, "rev-parse", "HEAD")
self.assertEqual(sm.append_only_problems(self.repo, self.commit, merged), [])
records = sm.read_records(self.repo, merged)
commit_all(self.tree, "right")
git(self.tree, "merge", "-q", "--no-edit", "left")
self.assertEqual(mr.append_only_problems(self.tree, self.commit), [])
records = mr.read_records(self.tree)
self.assertEqual(sum(1 for s in records if s.emulator == "alpha"), 4)
class CommittedRecords(unittest.TestCase):
"""The records this repository holds."""
def test_no_line_or_data_file_changed_or_vanished_since_head(self) -> None:
if subprocess.run(["git", "-C", str(REPO_ROOT), "rev-parse", "--verify", "-q", "HEAD"],
capture_output=True, check=False).returncode != 0:
self.skipTest("not a git checkout")
self.assertEqual(mr.append_only_problems(REPO_ROOT, "HEAD"), [])
def test_every_record_reads_and_joins_its_data(self) -> None:
records = mr.read_records(REPO_ROOT)
self.assertGreaterEqual(len(records), 7)
def test_jsonl_files_merge_by_union(self) -> None:
rules = (REPO_ROOT / ".gitattributes").read_text().splitlines()
rule = next(line for line in rules if line.startswith(f"{mr.RECORDS_DIR}/*.jsonl "))
self.assertIn("merge=union", rule.split())
class SitePages(unittest.TestCase):
ITEM = {
"emulator": "beetle_saturn", "emulator_name": "Beetle Saturn", "profiled": True,
@@ -376,14 +412,14 @@ class SitePages(unittest.TestCase):
"host": "Test CPU, 12 CPUs", "core": {"library": "Beetle Saturn v1.32.1", "commit": "2ce77e2" * 5 + "abcde",
"file": "x.so", "sha256": SHA256, "repo": None, "build": "make"},
"frontend_commit": None, "id": "2026-10-27-43527b763ace", "runs": 2,
"record_url": f"{REPO_URL}/blob/{'e' * 40}/records/beetle_saturn.jsonl#L4",
"data_url": f"{REPO_URL}/blob/{'e' * 40}/data/beetle_saturn/2026-10-27-43527b763ace.json.gz",
"record_url": f"{REPO_URL}/blob/{'e' * 40}/measurements/beetle_saturn.jsonl#L4",
"data_url": f"{REPO_URL}/blob/{'e' * 40}/measurements/data/beetle_saturn/2026-10-27-43527b763ace.json.gz",
"sqlite": "downloads/measurements/beetle_saturn.sqlite",
}
def test_tested_block(self) -> None:
self.assertEqual(sitemeasurements.tested_section("beetle_saturn", []), [])
text = "\n".join(sitemeasurements.tested_section("beetle_saturn", [self.ITEM]))
self.assertEqual(sm.tested_section("beetle_saturn", []), [])
text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM]))
self.assertTrue(text.startswith("## Tested\n"))
for expected in ("**Result:** `same`", "**Method:** Twice per card. Every input fixed.", "**Ran on:** Test CPU, 12 CPUs",
self.ITEM["record_url"], self.ITEM["data_url"], "1 flag raised", "2 runs",
@@ -392,10 +428,10 @@ class SitePages(unittest.TestCase):
self.assertIn(expected, text)
def test_hosts_table_on_the_method_page(self) -> None:
page = sitemeasurements.with_hosts(f"x\n\n{sitemeasurements.HOSTS_MARKER}\n",
page = sm.with_hosts(f"x\n\n{sm.HOSTS_MARKER}\n",
{"items": [self.ITEM, {**self.ITEM, "host": "Test CPU, 12 CPUs, load 1 before, 2 after"}]})
self.assertIn("| Test CPU, 12 CPUs | 2 |", page)
self.assertNotIn(sitemeasurements.HOSTS_MARKER, page)
self.assertNotIn(sm.HOSTS_MARKER, page)
def test_the_explorer_reads_the_index(self) -> None:
source = (REPO_ROOT / "website" / "src" / "components" / "MeasurementsExplorer.astro").read_text()
@@ -404,37 +440,44 @@ class SitePages(unittest.TestCase):
read = set(re.findall(r"\b(?:item|i)\.([a-z_]+)", source)) - {"length"}
self.assertTrue(read)
self.assertLessEqual(read, set(self.ITEM), read - set(self.ITEM))
page = sitemeasurements.generate_explorer_page({"records": 7, "count": 7, "commit": "e" * 40,
page = sm.generate_explorer_page({"records": 7, "count": 7, "commit": "e" * 40,
"repo": REPO_URL})
self.assertIn("<MeasurementsExplorer />", page)
self.assertIn("downloads/measurements.sqlite", page)
self.assertIn(f"({REPO_URL}/tree/{'e' * 40}/measurements)", page)
def test_the_deploy_clones_the_measurements_at_the_pin(self) -> None:
workflow = (REPO_ROOT / ".github" / "workflows" / "deploy-site.yml").read_text()
clone = workflow.index("Clone the measurements at their pin")
step = workflow[clone:workflow.index("- name:", clone)]
self.assertIn("measurements-pin.json", step)
self.assertIn('fetch -q --depth 1 "$repo" "$commit"', step)
self.assertIn('checkout -q --detach "$commit"', step)
self.assertNotIn("uses:", step)
self.assertLess(clone, workflow.index("Validate data contracts"))
self.assertLess(clone, workflow.index("python scripts/generate_site.py --require-measurements"))
self.assertIn('- "measurements-pin.json"', workflow)
pin = sm.load_pin()
self.assertEqual(pin["repo"], REPO_URL)
def test_the_builds_read_the_records_in_this_repository(self) -> None:
for name in ("deploy-site.yml", "validate.yml"):
workflow = (REPO_ROOT / ".github" / "workflows" / name).read_text()
with self.subTest(workflow=name):
self.assertIn('- "measurements/**"', workflow)
self.assertNotIn("measurements-pin", workflow)
self.assertNotIn(".cache/measurements", workflow)
self.assertNotIn("--require-measurements", workflow)
self.assertFalse((REPO_ROOT / "measurements-pin.json").exists())
self.assertFalse((REPO_ROOT / "scripts" / "sync_measurements.py").exists())
def test_nothing_links_the_former_measurements_repository(self) -> None:
former = "retrobios-" + "measurements"
roots = ["wiki", "schemas", "scripts", "website/src", ".github", "tests", "measurements"]
suffixes = {".md", ".py", ".json", ".jsonl", ".yml", ".astro", ".ts", ".mjs"}
stray = [path.relative_to(REPO_ROOT).as_posix()
for root in roots for path in sorted((REPO_ROOT / root).rglob("*"))
if path.is_file() and path.suffix in suffixes
and former in path.read_text(encoding="utf-8", errors="replace")]
stray += [name for name in ("README.md", "CONTRIBUTING.md")
if former in (REPO_ROOT / name).read_text(encoding="utf-8")]
self.assertEqual(stray, [])
class Freshness(unittest.TestCase):
def test_pin_against_the_remote_head(self) -> None:
import check_freshness as cf # noqa: PLC0415
def test_no_measurements_area(self) -> None:
import check_freshness # noqa: PLC0415
import refresh_stale # noqa: PLC0415
pin = sm.load_pin()
self.assertEqual(cf.check_measurements(False, head=lambda _repo: pin["commit"]).status, cf.OK)
stale = cf.check_measurements(False, head=lambda _repo: "f" * 40)
self.assertEqual(stale.status, cf.STALE)
self.assertIn("sync_measurements.py --fetch --bump", stale.detail)
self.assertEqual(cf.check_measurements(True).status, cf.SKIPPED)
self.assertIn("measurements", cf.AREAS)
self.assertNotIn("measurements", check_freshness.AREAS)
self.assertNotIn("measurements", refresh_stale.AUTO_AREAS)
if __name__ == "__main__":
tests/test_site_r4_reference.py
+10 -19
View File
@@ -248,32 +248,23 @@ class SourceLinksOnlyWhereTheFileIs(unittest.TestCase):
def test_every_rendered_link_is_recorded(self):
record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8"))
pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*"))
import sync_measurements # noqa: PLC0415
import measurement_records # noqa: PLC0415
# A Tested block links the measurements repository at its pin. The
# record of source trees does not cover it: those paths are checked
# against the pinned tree of the clone itself.
measured_slug = urllib.parse.urlsplit(sync_measurements.load_pin()["repo"]).path.strip("/")
measured: dict[str, set[str]] = {}
# A Tested block links a record line and its data file in this
# repository. The records are append-only, so a file a permalink
# names is still in the tree.
records = set(measurement_records.tree_files(REPO_ROOT))
checked, stray = 0, []
for page in pages:
for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")):
if "/Abdess/retrobios/" in found[0]:
continue
key = f"{found['host']}/{found['slug']}@{found['pin']}"
path = urllib.parse.unquote(found["path"])
checked += 1
if found["slug"] == measured_slug:
if found["pin"] not in measured:
if not sync_measurements.has_commit(sync_measurements.CLONE, found["pin"]):
stray.append((page.name, key, "pin not in the measurements clone"))
measured[found["pin"]] = set()
continue
measured[found["pin"]] = set(
sync_measurements.tree_files(sync_measurements.CLONE, found["pin"]))
if path not in measured[found["pin"]]:
if "/Abdess/retrobios/" in found[0]:
if path.startswith(f"{measurement_records.RECORDS_DIR}/") and path not in records:
stray.append((page.name, key, path))
elif path not in record.get(key, ()):
continue
checked += 1
if path not in record.get(key, ()):
stray.append((page.name, key, path))
self.assertGreater(checked, 1000)
self.assertEqual(stray[:20], [])
website/src/components/MeasurementsExplorer.astro
+1 -1
View File
@@ -1,7 +1,7 @@
---
// The Measurements page: every test of the measurements index, filtered in
// the browser. Reads public/data/measurements/index.json, which
// scripts/sync_measurements.py writes from the records at the pinned commit.
// scripts/sitemeasurements.py writes from the records under measurements/.
// Each row keeps its method and the computer it ran on beside the result,
// and links its full record. The filters answer from the index alone: no
// SQL engine is loaded.