From 571e50f2a89c45f22a9c4196ea7238e0d30cc19a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:11:12 +0100 Subject: [PATCH] refactor: read measurements from the local tree The site compiles the index and the SQLite files from measurements/ instead of a pinned clone. The pin, the sync script, the CI clone steps, the freshness area and its refresh handler are gone. core_probe --record writes into measurements/ of the checkout by default, and a test fails when a line or data file HEAD holds changed or vanished. --- .github/workflows/deploy-site.yml | 17 +- .github/workflows/validate.yml | 16 +- measurements-pin.json | 4 - schemas/core-probe-data.schema.json | 2 +- schemas/core-probe-record.schema.json | 4 +- scripts/check_freshness.py | 34 +- scripts/core_probe.py | 44 +- scripts/generate_site.py | 14 +- scripts/measurement_records.py | 126 +++- scripts/refresh_stale.py | 10 +- scripts/sitemeasurements.py | 370 +++++++++++- scripts/sync_measurements.py | 564 ------------------ scripts/validate_schemas.py | 41 +- tests/test_core_probe.py | 26 +- tests/test_measurements.py | 277 +++++---- tests/test_site_r4_reference.py | 29 +- .../src/components/MeasurementsExplorer.astro | 2 +- 17 files changed, 699 insertions(+), 881 deletions(-) delete mode 100644 measurements-pin.json delete mode 100644 scripts/sync_measurements.py diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 0dd7ecbd9..f73a52a08 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -57,7 +57,6 @@ on: - "scripts/slots.py" - "scripts/sourceref.py" - "scripts/split_pack.py" - - "scripts/sync_measurements.py" - "scripts/truth.py" - "scripts/upstream.py" - "scripts/validate_schemas.py" @@ -72,7 +71,7 @@ on: - "database.json" - "release.json" - "derivable.json" - - "measurements-pin.json" + - "measurements/**" # The Starlight project: config, components, styles, lockfile. Its # generated folders are gitignored and never trigger anything. - "website/**" @@ -118,18 +117,6 @@ jobs: - run: npm ci --prefix website - # The core probe records live in Abdess/retrobios-measurements, read - # at the commit measurements-pin.json names and never at its head: a - # record reaches the site when the pin moves. One shallow fetch of that - # commit, with git alone. - - name: Clone the measurements at their pin - run: | - repo=$(python -c "import json; print(json.load(open('measurements-pin.json'))['repo'])") - commit=$(python -c "import json; print(json.load(open('measurements-pin.json'))['commit'])") - git init -q .cache/measurements - git -C .cache/measurements fetch -q --depth 1 "$repo" "$commit" - git -C .cache/measurements checkout -q --detach "$commit" - - name: Validate data contracts run: python scripts/validate_schemas.py @@ -149,7 +136,7 @@ jobs: - name: Generate site run: | - python scripts/generate_site.py --require-measurements + python scripts/generate_site.py python scripts/generate_readme.py --db database.json --platforms-dir platforms # starlight-links-validator fails the build on a dead link or anchor. npm run build --prefix website diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 1e2910632..775f7207a 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -29,7 +29,7 @@ on: - "database.json" - "release.json" - "derivable.json" - - "measurements-pin.json" + - "measurements/**" - "website/**" # Spelled out twice because the workflow parser reads no YAML anchor. A test # holds the two lists equal. @@ -54,7 +54,7 @@ on: - "database.json" - "release.json" - "derivable.json" - - "measurements-pin.json" + - "measurements/**" - "website/**" permissions: @@ -259,21 +259,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # The measurement records at their pin, as deploy-site.yml reads them: - # the pages and the schema check read the same lines. - - name: Clone the measurements at their pin - run: | - repo=$(python -c "import json; print(json.load(open('measurements-pin.json'))['repo'])") - commit=$(python -c "import json; print(json.load(open('measurements-pin.json'))['commit'])") - git init -q .cache/measurements - git -C .cache/measurements fetch -q --depth 1 "$repo" "$commit" - git -C .cache/measurements checkout -q --detach "$commit" - # The pages are gitignored, and the tests that read them fail # in CI without it. The suite runs whatever this step returned, so a # generator that stops still leaves every other test reporting. - name: Generate the site - run: python scripts/generate_site.py --require-measurements + run: python scripts/generate_site.py - name: Run test suite if: ${{ !cancelled() }} diff --git a/measurements-pin.json b/measurements-pin.json deleted file mode 100644 index 4fe2930d5..000000000 --- a/measurements-pin.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "commit": "75e878143687201d1d7ca34f09cb43be3dc8d855", - "repo": "https://github.com/Abdess/retrobios-measurements" -} diff --git a/schemas/core-probe-data.schema.json b/schemas/core-probe-data.schema.json index bbd90624b..a902d7451 100644 --- a/schemas/core-probe-data.schema.json +++ b/schemas/core-probe-data.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-data.schema.json", "title": "RetroBIOS core probe record data", - "description": "The sidecar of a core probe record, data//.json.gz: every repetition in full, the raw timing values and compared values too large for the record line. With the record it rebuilds the full result.", + "description": "The sidecar of a core probe record, measurements/data//.json.gz: every repetition in full, the raw timing values and compared values too large for the record line. With the record it rebuilds the full result.", "type": "object", "additionalProperties": false, "required": [ diff --git a/schemas/core-probe-record.schema.json b/schemas/core-probe-record.schema.json index d8669c056..f969c9104 100644 --- a/schemas/core-probe-record.schema.json +++ b/schemas/core-probe-record.schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-record.schema.json", "title": "RetroBIOS core probe record", - "description": "One line of records/.jsonl in the measurements repository: one run of scripts/core_probe.py, a libretro core executed headless, with what is needed to judge and reproduce it. The bulky rest (every repetition, core logs, raw timings) is the gzip sidecar data//.json.gz, valid against core-probe-data.schema.json, whose sha256 is in `data`.", + "description": "One line of measurements/.jsonl: one run of scripts/core_probe.py, a libretro core executed headless, with what is needed to judge and reproduce it. The bulky rest (every repetition, core logs, raw timings) is the gzip sidecar measurements/data//.json.gz, valid against core-probe-data.schema.json, whose sha256 is in `data`.", "type": "object", "additionalProperties": false, "required": [ @@ -32,7 +32,7 @@ }, "emulator": { "$ref": "#/$defs/slug", - "description": "The emulator profile the record is filed under: records/.jsonl." + "description": "The emulator profile the record is filed under: measurements/.jsonl." }, "subject": { "$ref": "#/$defs/slug", diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index f06ec170b..a1116142d 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -3,7 +3,7 @@ Each layer the repository transcribes moves on its own schedule: platform lists, hardware targets, core-info, buildbot system assets, dump catalogs, -romset DATs, the measurements pin, the CI toolchain. This script asks each of them the same +romset DATs, the CI toolchain. This script asks each of them the same question, is the local copy the one upstream serves, and answers with one row per subject. @@ -40,7 +40,7 @@ import shutil import subprocess import sys import tempfile -from collections.abc import Callable, Iterator +from collections.abc import Iterator from concurrent.futures import ThreadPoolExecutor from dataclasses import asdict, dataclass, field from datetime import datetime, timezone @@ -80,8 +80,7 @@ INSTALLERS = ("install.sh", "install.ps1") SHOWN_ITEMS = 4 # items named in a diff summary before ", ..." SHOWN_NAMES = 12 # core names named in an unresolved row before ", ..." -AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "measurements", "ci", - "profiles") +AREAS = ("platforms", "native", "targets", "coreinfo", "data", "catalogs", "ci", "profiles") OK, STALE, UNKNOWN, ERROR, SKIPPED = "OK", "STALE", "UNKNOWN", "ERROR", "SKIPPED" # Every status a finding may carry, in the order the summary counts them. STATUSES = (STALE, UNKNOWN, ERROR, SKIPPED, OK) @@ -915,31 +914,6 @@ def _check_buildbot_system() -> Finding: detail=f"{names}{more} (_data_dirs.yml)", items=every) -# --- measurements ----------------------------------------------------------- - - -def check_measurements(offline: bool, head: Callable[[str], str] | None = None, - pin_path: Path | None = None) -> Finding: - """The measurements pin against the head of the measurements repository. - - A record added there reaches the site only once the pin moves to it. - """ - import sync_measurements # noqa: PLC0415 - - pin = sync_measurements.load_pin(pin_path or sync_measurements.PIN_FILE) - local = pin["commit"][:12] - if offline: - return Finding("measurements", "records pin", SKIPPED, local=local, detail="offline") - try: - remote = (head or sync_measurements.remote_head)(pin["repo"]) - except (sync_measurements.MeasurementsError, OSError, subprocess.TimeoutExpired) as exc: - return Finding("measurements", "records pin", UNKNOWN, local=local, detail=str(exc)) - if remote == pin["commit"]: - return Finding("measurements", "records pin", OK, local=local, remote=remote[:12]) - return Finding("measurements", "records pin", STALE, local=local, remote=remote[:12], - detail="newer records: python scripts/sync_measurements.py --fetch --bump") - - # --- catalogs and recipes --------------------------------------------------- @@ -1532,8 +1506,6 @@ def main() -> int: findings.extend(check_data(args.offline)) if "catalogs" in areas: findings.extend(check_catalogs(args.cache_dir, args.offline)) - if "measurements" in areas: - findings.append(check_measurements(args.offline)) if "ci" in areas: findings.extend(check_ci(args.cache_dir, args.offline)) if "profiles" in areas: diff --git a/scripts/core_probe.py b/scripts/core_probe.py index 1f8d3f3ff..62ff1e5de 100644 --- a/scripts/core_probe.py +++ b/scripts/core_probe.py @@ -10,10 +10,10 @@ Three modes share one runner: Each repetition runs in its own process, under strace when it is installed, so the files a core opened are read from the kernel rather than guessed. -`--record DIR` appends one record per invocation to a clone of the -measurements repository, carrying what is needed to reproduce it (core build, -frontend commit, host, inputs by hash). measurement_records.py splits it into -a slim line and a gzip sidecar. +`--record` appends one record per invocation to measurements/ of this +repository, carrying what is needed to reproduce it (core build, frontend +commit, host, inputs by hash). measurement_records.py splits it into a slim +line and a gzip sidecar. """ from __future__ import annotations @@ -39,7 +39,6 @@ from dataclasses import dataclass, field from pathlib import Path ROOT = Path(__file__).resolve().parent.parent -MEASUREMENTS_CLONE = ROOT / ".cache" / "measurements" MIN_BENCH_REPETITIONS = 5 DEFAULT_WORK_DIR = ROOT / "tmp" / "core_probe" BASE_OBSERVABLES = ("frame_sha1", "video_sha1", "audio_sha1", "state_sha1") @@ -771,10 +770,10 @@ def build_parser() -> argparse.ArgumentParser: common.add_argument("--core-repo", help="source repository URL of the core build") common.add_argument("--core-commit", help="commit the core was built from") common.add_argument("--build-command", help="command and flags that built the core") - common.add_argument("--record", metavar="DIR", - help="append the record to the measurements clone DIR " - f"(records/.jsonl and data//), " - f"for example {MEASUREMENTS_CLONE.relative_to(ROOT)}") + common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT), + help="append the record to measurements/.jsonl and write " + "its data file in measurements/data// of the checkout " + "DIR (default: this one)") common.add_argument("--emulator", help="profile the record is filed under (default: the " "profile whose cores: names the .so, else the " "core's library name)") @@ -823,7 +822,6 @@ def validate_args(args: argparse.Namespace) -> list[str]: if args.png_dir and not args.frame_hash_every: problems.append("--png-dir needs --frame-hash-every") if args.record: - problems.extend(record_target_problems(Path(args.record))) if not args.emulator and Path(args.core).is_file(): try: default_emulator(Path(args.core), None) @@ -907,32 +905,6 @@ def check_result(variants: list[dict], observable: str) -> dict: } -def record_target_problems(directory: Path) -> list[str]: - """Why records may not be written under ``directory``. - - Records live in their own repository. A directory inside this one's - tracked tree would put them in its history, megabytes per campaign; - an ignored directory (tmp/, the clone under .cache/) is fine. - """ - target = directory.resolve() - if target != ROOT and ROOT not in target.parents: - return [] - try: - ignored = subprocess.run(["git", "check-ignore", "-q", str(target)], cwd=ROOT, - capture_output=True, check=False).returncode - except OSError as exc: - return [f"--record {directory}: cannot ask git whether it is tracked ({exc})"] - if ignored == 0: - return [] - if ignored != 1: - # 128: not a git checkout, nothing to keep records out of. - return [] - return [f"--record {directory}: inside the tracked tree of this repository. Records go " - f"to the measurements repository: clone it with " - f"`python scripts/sync_measurements.py --fetch` and pass " - f"--record {MEASUREMENTS_CLONE.relative_to(ROOT)}"] - - def default_emulator(core_file: Path, library_name: str | None) -> str: """The profile whose cores: names the .so, else the library name as a slug.""" stem = core_file.name.split(".")[0].removesuffix("_libretro") diff --git a/scripts/generate_site.py b/scripts/generate_site.py index eaf3dfc57..dd04f5697 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -110,7 +110,6 @@ from siteguides import ( ) import release_record import sitemeasurements -import sync_measurements # The Starlight project, and where its pages go inside it. STARLIGHT_DIR = "website" @@ -11007,11 +11006,6 @@ def main() -> None: help="directory the pages are written to (default: the project's " "content collection, src/content/docs)", ) - parser.add_argument( - "--require-measurements", action="store_true", - help="fail when the measurements clone does not hold the pinned commit " - "(the deploy build passes it; without it the site lists no measurements)", - ) args = parser.parse_args() if args.docs_dir is None: args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT) @@ -11168,9 +11162,9 @@ def _generate(args: argparse.Namespace) -> None: if clash: raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}") - print("Compiling the measurements at their pin...") - measured = sync_measurements.build( - public, emulators_dir=Path(args.emulators_dir), require=args.require_measurements, + print("Compiling the measurements...") + measured = sitemeasurements.build( + public, emulators_dir=Path(args.emulators_dir), repo=REPO_URL, system_names={sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))}, ) for path in measured.paths: @@ -11187,7 +11181,7 @@ def _generate(args: argparse.Namespace) -> None: ) write_release_advertised(Path(args.site_dir) / "src" / "generated") write_if_changed(str(docs / "data.md"), - generate_data_page(exports, public / sync_measurements.SQLITE_PATH) + generate_data_page(exports, public / sitemeasurements.SQLITE_PATH) + reference_next()) PAGE_DATA["data.md"] = dataset_structured_data(exports, db) diff --git a/scripts/measurement_records.py b/scripts/measurement_records.py index 0b135bf1a..ff57bab5b 100644 --- a/scripts/measurement_records.py +++ b/scripts/measurement_records.py @@ -1,9 +1,9 @@ -"""Core probe records as the measurements repository stores them. +"""Core probe records as this repository stores them. A run of core_probe.py produces one full record. It is stored as two parts: - records/.jsonl one slim record per line, append-only - data//.json.gz the bulky rest, named by the record id + measurements/.jsonl one slim record per line, append-only + measurements/data//.json.gz the bulky rest, named by the record id The slim record keeps what a reader needs to judge and reproduce the result: claim, method, observable, result, statistics, host, inputs by hash, core @@ -18,7 +18,7 @@ A record id is the date and the first 12 hex digits of the sha256 of the record without its id. A record carried over from the first layout keeps the id it had, which hashes that full record. -Standard library only: core_probe.py, sync_measurements.py and the site +Standard library only: core_probe.py, validate_schemas.py and the site read it. """ @@ -29,12 +29,14 @@ import gzip import hashlib import json import re +import subprocess from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path RECORD_VERSION = 2 -MEASUREMENTS_REPO = "https://github.com/Abdess/retrobios-measurements" -RECORDS_DIR = "records" -DATA_DIR = "data" +RECORDS_DIR = "measurements" +DATA_DIR = f"{RECORDS_DIR}/data" # Fields a repetition keeps in the slim record, per variant, from the first one. VARIANT_SUMMARY = ("observables", "files_opened", "files_probed_absent", "tracing") # Observables too large for a line: their values stay in the sidecar. @@ -244,3 +246,113 @@ def live(records: Iterable[dict]) -> list[dict]: records = list(records) superseded = {r["supersedes"] for r in records if r.get("supersedes")} return [r for r in records if r["id"] not in superseded] + + +class MeasurementsError(RuntimeError): + """The records of a tree cannot be read, or break a rule.""" + + +@dataclass(frozen=True) +class Stored: + """One record line as read from the tree.""" + + emulator: str + line: int + record: dict + + +def tree_files(root: Path) -> dict[str, Path]: + """Repository path -> file of every record file and data file under ``root``.""" + base = root / RECORDS_DIR + if not base.is_dir(): + return {} + return {path.relative_to(root).as_posix(): path + for path in sorted(base.rglob("*")) if path.is_file()} + + +def read_records(root: Path) -> list[Stored]: + """Every record line under ``root``, each checked against its rules and data.""" + files = tree_files(root) + data_prefix = f"{DATA_DIR}/" + problems: list[str] = [] + stored: list[Stored] = [] + claimed: set[str] = set() + for path in sorted(p for p in files if not p.startswith(data_prefix)): + emulator = path[len(RECORDS_DIR) + 1:].removesuffix(".jsonl") + if not path.endswith(".jsonl") or "/" in emulator: + problems.append(f"{path}: {RECORDS_DIR}/ holds .jsonl files and data/ only") + continue + seen: set[str] = set() + try: + records = parse_lines(files[path].read_text(encoding="utf-8"), path) + except RecordError as exc: + problems.append(str(exc)) + continue + for number, record in enumerate(records, 1): + where = f"{path}:{number}" + problems.extend(f"{where}: {p}" for p in check_line(record)) + if record.get("emulator") != emulator: + problems.append(f"{where}: emulator {record.get('emulator')!r} in {path}") + if record.get("id") in seen: + problems.append(f"{where}: id {record.get('id')} appears twice") + if record.get("supersedes") and record["supersedes"] not in seen: + problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file") + seen.add(record.get("id")) + data = sidecar_path(emulator, str(record.get("id"))) + claimed.add(data) + if data not in files: + problems.append(f"{where}: {data} is missing") + else: + problems.extend(f"{where}: {p}" for p in check_pair(record, files[data].read_bytes())) + stored.append(Stored(emulator, number, record)) + problems.extend(f"{path}: no record line names it" + for path in sorted(files) if path.startswith(data_prefix) and path not in claimed) + if problems: + raise MeasurementsError("records under {}:\n {}".format(root, "\n ".join(problems))) + return stored + + +def _git(root: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run(["git", "-C", str(root), *args], capture_output=True, check=False) + + +def committed_files(root: Path, ref: str) -> dict[str, bytes]: + """Path -> content of every record and data file at ``ref``.""" + listing = _git(root, "ls-tree", "-r", "-z", "--name-only", ref, "--", RECORDS_DIR) + if listing.returncode != 0: + raise MeasurementsError(f"git ls-tree {ref}: {listing.stderr.decode(errors='replace').strip()}") + out = {} + for path in filter(None, listing.stdout.decode("utf-8").split("\0")): + blob = _git(root, "show", f"{ref}:{path}") + if blob.returncode != 0: + raise MeasurementsError(f"git show {ref}:{path}: {blob.stderr.decode(errors='replace').strip()}") + out[path] = blob.stdout + return out + + +def append_only_problems(root: Path, ref: str = "HEAD") -> list[str]: + """What the tree under ``root`` changed or removed of the lines and data + files ``ref`` holds. Lines added at the end and new files are allowed.""" + before, after = committed_files(root, ref), tree_files(root) + problems = [] + for path, content in sorted(before.items()): + if path not in after: + problems.append(f"{path}: removed") + continue + now = after[path].read_bytes() + if now == content: + continue + if path.endswith(".jsonl"): + moved = appended_only(content.decode("utf-8"), now.decode("utf-8")) + if moved: + problems.append(f"{path}: {moved}") + else: + problems.append(f"{path}: rewritten") + return problems + + +def last_commit(root: Path) -> str | None: + """The last commit that touched the records: the one their permalinks name.""" + proc = _git(root, "log", "-1", "--format=%H", "--", RECORDS_DIR) + commit = proc.stdout.decode().strip() if proc.returncode == 0 else "" + return commit if re.fullmatch(r"[0-9a-f]{40}", commit) else None diff --git a/scripts/refresh_stale.py b/scripts/refresh_stale.py index 6d4f35570..d79b6488d 100755 --- a/scripts/refresh_stale.py +++ b/scripts/refresh_stale.py @@ -27,9 +27,6 @@ Mapping (stale -> command): --source mame --fetch catalogs/fbneo recipes python -m scripts.scraper.romset_dat_importer --source fbneo --fetch - measurements/records pin - python scripts/sync_measurements.py --fetch --bump, - which refuses a remote that rewrote a pinned line A platform that inherits from a rescraped one (retropie and lakka from retroarch) reads the parent's file: its jobs start once the parent's chain @@ -66,7 +63,7 @@ CHECK_FRESHNESS = REPO_ROOT / "scripts" / "check_freshness.py" PLATFORMS_REGISTRY = REPO_ROOT / "platforms" / "_registry.yml" DATA_DIRS_REGISTRY = REPO_ROOT / "platforms" / "_data_dirs.yml" -AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs", "measurements") +AUTO_AREAS = ("platforms", "native", "targets", "data", "catalogs") MANUAL_AREAS = ("coreinfo", "ci", "profiles") JOB_TIMEOUT = 1800 # 30 minutes per refresher, scrapers rarely exceed 10 @@ -410,11 +407,6 @@ def _command_for( # redump / no-intro / tosec need manual DAT download. return None - if area == "measurements": - # --fetch refuses a remote that rewrote a pinned line, and --bump - # only rewrites measurements-pin.json: nothing is committed. - return [sys.executable, "scripts/sync_measurements.py", "--fetch", "--bump"] - # coreinfo, ci, profiles: never auto-refresh. return None diff --git a/scripts/sitemeasurements.py b/scripts/sitemeasurements.py index 369ac34ed..22862f2f8 100644 --- a/scripts/sitemeasurements.py +++ b/scripts/sitemeasurements.py @@ -1,24 +1,375 @@ -"""The site's measurement pages, from the index sync_measurements.py writes. +"""The measurements the site shows, from the records under measurements/. -Three places read the same index: the Tested block of each emulator page, -the Measurements explorer page and the host table of the methodology page. -A number is never shown without its method and host beside it. +The records are core probe runs, one append-only .jsonl per emulator with a +gzip data file per record (measurement_records.py). From them the build +writes, never committed: + + public/data/measurements/index.json the compact index the explorer and + the emulator pages read, one item + per emulator and subject + public/downloads/measurements.sqlite every record, normalised + public/downloads/measurements/.sqlite one database per emulator + +Three places read the index: the Tested block of each emulator page, the +Measurements explorer page and the host table of the methodology page. A +number is never shown without its method and host beside it. A record links +its line at the last commit that touched measurements/. """ from __future__ import annotations +import json +import os import re import sqlite3 +import sys +from dataclasses import dataclass from pathlib import Path +import measurement_records as mr import sitemarkup as markup +from artifacts import json_text, write_if_changed from siterender import _escape_tags +ROOT = Path(__file__).resolve().parent.parent +REPO = "https://github.com/Abdess/retrobios" EXPLORER_PAGE = "measurements.md" METHOD_PAGE = "wiki/measurements.md" HOSTS_MARKER = "" +INDEX_PATH = Path("data") / "measurements" / "index.json" +SQLITE_PATH = Path("downloads") / "measurements.sqlite" +PER_EMULATOR_DIR = Path("downloads") / "measurements" +SQLITE_SCHEMA_VERSION = "1" +INDEX_SCHEMA_VERSION = 1 +# -------------------------------------------------------------------------- +# Index +# -------------------------------------------------------------------------- + +def host_summary(host: dict) -> str: + parts = [host.get("cpu_model") or "unknown CPU"] + if host.get("cpu_count"): + parts.append(f"{host['cpu_count']} CPUs") + parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip() + if "Linux" in host.get("os", "") else host.get("os", "")) + parts.append(f"Python {host.get('python', '')}") + before, after = host.get("load_before"), host.get("load_after") + if before and after: + parts.append(f"load {before[0]} before, {after[0]} after") + return ", ".join(part for part in parts if part) + + +def bench_statement(record: dict) -> str: + timing = record["timing"] + frame, fps = timing["ms_per_frame"], timing["fps"] + return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), " + f"min {frame['min']:.3f}, max {frame['max']:.3f}, over " + f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames") + + +def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str: + return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "") + + +def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict, + system_names: dict[str, str]) -> dict: + record = entry.record + profile = profiles.get(entry.emulator) or {} + check = record.get("check") or {} + core = record["core"] + content = (record["inputs"].get("content") or {}) + return { + "emulator": entry.emulator, + "emulator_name": str(profile.get("emulator", entry.emulator)), + "profiled": bool(profile), + "systems": sorted(set(profile.get("systems") or [])), + "system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))], + "subject": record["subject"], + "kind": record["kind"], + "claim": check.get("claim"), + "method": check.get("method"), + "observable": check.get("observable"), + "variable": check.get("variable"), + "status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"), + "statement": (check.get("result") or {}).get("statement") + or (bench_statement(record) if record.get("timing") else ""), + "content": content.get("label"), + "variants": len(record["variants"]), + "repetitions": record["inputs"]["repetitions"], + "flags": len(record["flags"]), + "date": record["recorded_at"], + "host": host_summary(record["host"]), + "core": { + "library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])), + "file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"), + "commit": core.get("commit"), "build": core.get("build_command"), + }, + "frontend_commit": record["frontend"].get("commit"), + "id": record["id"], + "runs": runs, + "record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line), + "data_url": permalink(source["repo"], source["commit"], mr.sidecar_path(entry.emulator, record["id"])), + "sqlite": f"downloads/measurements/{entry.emulator}.sqlite", + } + + +def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict, + system_names: dict[str, str] | None = None) -> dict: + """One item per (emulator, subject): its latest record not superseded.""" + by_subject: dict[tuple[str, str], list[mr.Stored]] = {} + live_ids = {r["id"] for r in mr.live(s.record for s in stored)} + for entry in stored: + if entry.record["id"] in live_ids: + by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry) + items = [] + for key in sorted(by_subject): + entries = by_subject[key] + latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line)) + items.append(index_item(latest, len(entries), source or {}, profiles, system_names or {})) + items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"])) + return { + "schema_version": INDEX_SCHEMA_VERSION, + "kind": "measurements", + "repo": (source or {}).get("repo"), + "commit": (source or {}).get("commit"), + "records": len(stored), + "count": len(items), + "items": items, + } + + +# -------------------------------------------------------------------------- +# SQLite +# -------------------------------------------------------------------------- + +SCHEMA_SQL = """ + PRAGMA journal_mode = OFF; + PRAGMA synchronous = OFF; + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE cores ( + sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT, + build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT + ); + CREATE TABLE hosts ( + id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL, + kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL + ); + CREATE TABLE emulator_systems ( + emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system) + ); + CREATE TABLE runs ( + id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL, + kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT, + superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256), + host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT, + max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL, + script_modified INTEGER, flags TEXT NOT NULL, data_sha256 TEXT NOT NULL, + data_bytes INTEGER NOT NULL, record_url TEXT NOT NULL, line INTEGER NOT NULL, + migrated_from_version INTEGER + ); + CREATE TABLE inputs ( + run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT, + content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL, + warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL, + observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL + ); + CREATE TABLE content_tracks ( + run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL, + file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL, + PRIMARY KEY (run_id, position) + ); + CREATE TABLE checks ( + run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL, + observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL, + statement TEXT NOT NULL + ); + CREATE TABLE check_groups ( + run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL, + value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position) + ); + CREATE TABLE variants ( + run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL, + label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL, + flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT, + files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position) + ); + CREATE TABLE system_files ( + run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL, + destination TEXT NOT NULL, sha1 TEXT, + PRIMARY KEY (run_id, variant_position, destination) + ); + CREATE TABLE benchmarks ( + run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL, + median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL, + min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric) + ); + CREATE INDEX runs_emulator_idx ON runs(emulator, subject); + CREATE INDEX runs_subject_idx ON runs(subject); + CREATE INDEX emulator_systems_system_idx ON emulator_systems(system); + CREATE INDEX checks_claim_idx ON checks(claim); + CREATE INDEX system_files_sha1_idx ON system_files(sha1); +""" + +TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks", + "checks", "check_groups", "variants", "system_files", "benchmarks") + + +def _host_key(host: dict) -> tuple: + return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel", + "machine", "python")) + + +def write_sqlite(destination: Path, stored: list[mr.Stored], source: dict | None, profiles: dict, + scratch: Path) -> None: + """A deterministic database of the records: sorted rows, no clock.""" + scratch.mkdir(parents=True, exist_ok=True) + temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite" + temp.unlink(missing_ok=True) + destination.parent.mkdir(parents=True, exist_ok=True) + stored = sorted(stored, key=lambda s: (s.emulator, s.line)) + live_ids = {r["id"] for r in mr.live(s.record for s in stored)} + hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)} + connection = sqlite3.connect(temp) + try: + connection.executescript(SCHEMA_SQL) + meta = { + "schema_version": SQLITE_SCHEMA_VERSION, + "record_version": str(mr.RECORD_VERSION), + "measurements_repo": (source or {}).get("repo") or "", + "measurements_commit": (source or {}).get("commit") or "", + "license": "MIT", + "scope": "core probe records: the record lines, not their data files", + } + connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items())) + cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored} + connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ + (sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")), + json_text(core.get("build_command")), json_text(core.get("compiler")), + json_text(core.get("library_name")), json_text(core.get("library_version"))) + for sha, core in sorted(cores.items())]) + connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [ + (number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])]) + emulators = sorted({s.emulator for s in stored}) + connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [ + (emu, system) for emu in emulators + for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))]) + for entry in stored: + _insert_run(connection, entry, source or {}, hosts, entry.record["id"] not in live_ids) + connection.commit() + connection.execute("VACUUM") + finally: + connection.close() + os.replace(temp, destination) + + +def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict, + hosts: dict, superseded: bool) -> None: + r = entry.record + host, front, inputs = r["host"], r["frontend"], r["inputs"] + rid = r["id"] + connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( + rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"), + int(superseded), r["core"]["sha256"], hosts[_host_key(host)], + json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"), + front.get("commit"), front["script_sha256"], + None if front.get("script_modified") is None else int(front["script_modified"]), + json_text(r["flags"]), r["data"]["sha256"], r["data"]["bytes"], + permalink(source.get("repo", ""), source.get("commit", ""), mr.records_path(entry.emulator), entry.line), + entry.line, r.get("migrated_from_version"))) + content = inputs.get("content") or {} + connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( + rid, content.get("label"), content.get("sha1"), content.get("size"), + json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"], + inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]), + inputs["frame_hash_every"], inputs["joypad_ports"])) + connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [ + (rid, position, track["file"], track["sha1"], track["size"]) + for position, track in enumerate(content.get("tracks") or [])]) + check = r.get("check") + if check: + result = check["result"] + connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", ( + rid, check["claim"], check["method"], check["observable"], check["variable"], + result["status"], result["statement"])) + connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [ + (rid, position, json_text(group["value"]) if "value" in group else None, + group.get("value_sha1"), json_text(group["variants"])) + for position, group in enumerate(result["groups"])]) + for position, variant in enumerate(r["variants"]): + connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( + rid, position, variant["label"], variant["repetitions"], int(variant["loaded"]), + json_text(variant["flags"]), json_text(variant["observables"]), + json_text(variant["files_opened"]), json_text(variant["files_probed_absent"]), + json_text(variant["tracing"]))) + connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [ + (rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())]) + for metric, stats in sorted((r.get("timing") or {}).items()): + connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", ( + rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"])) + + +# -------------------------------------------------------------------------- +# Building the site's files +# -------------------------------------------------------------------------- + +@dataclass +class Built: + index: dict + paths: list[Path] + + +def load_profiles(emulators_dir: Path) -> dict: + from common import load_emulator_profiles # noqa: PLC0415 + + return load_emulator_profiles(str(emulators_dir), skip_aliases=False) + + +def records_commit(root: Path) -> str | None: + """The commit the record links name, with a warning when the records on + disk are not the ones it holds.""" + commit = mr.last_commit(root) + if commit is None: + print(f"WARNING: no commit holds {root / mr.RECORDS_DIR}: the record links " + "lead nowhere until the records are committed.", file=sys.stderr) + return None + on_disk = {path: file.read_bytes() for path, file in mr.tree_files(root).items()} + if mr.committed_files(root, commit) != on_disk: + print(f"WARNING: {mr.RECORDS_DIR}/ differs from {commit[:12]}, the last commit that " + "touched it: links to records added since lead nowhere until they are " + "committed.", file=sys.stderr) + return commit + + +def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulators", + scratch: Path = ROOT / "tmp" / "site", system_names: dict[str, str] | None = None, + repo: str = REPO, commit: str | None = None) -> Built: + """Write the index and the databases under the site's public folder. + + ``commit`` is the revision the record links name, by default the last + commit that touched the records. + """ + stored = mr.read_records(root) + source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)} + profiles = load_profiles(emulators_dir) + index = build_index(stored, source, profiles, system_names) + paths = [public / INDEX_PATH] + (public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True) + write_if_changed(str(public / INDEX_PATH), + json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n") + write_sqlite(public / SQLITE_PATH, stored, source, profiles, scratch) + paths.append(public / SQLITE_PATH) + for emulator in sorted({s.emulator for s in stored}): + target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite" + write_sqlite(target, [s for s in stored if s.emulator == emulator], source, profiles, scratch) + paths.append(target) + return Built(index, paths) + + +# -------------------------------------------------------------------------- +# Pages +# -------------------------------------------------------------------------- + def by_emulator(index: dict) -> dict[str, list[dict]]: out: dict[str, list[dict]] = {} for item in index.get("items", []): @@ -91,7 +442,7 @@ def hosts_table(index: dict) -> list[str]: computer = item["host"].split(", load ")[0] hosts[computer] = hosts.get(computer, 0) + 1 if not hosts: - return ["No result is recorded at the pinned commit yet.", ""] + return ["No result is recorded yet.", ""] return [ "| Computer | Tests |", "|----------|------:|", @@ -108,8 +459,8 @@ def with_hosts(page: str, index: dict) -> str: def generate_explorer_page(index: dict) -> str: commit = index.get("commit") repo = index.get("repo") or "" - source = (f"[retrobios-measurements]({repo}/tree/{commit}) at {_short(commit)}" - if commit else "the measurements project, not read in this build") + source = (f"[`measurements/`]({repo}/tree/{commit}/{mr.RECORDS_DIR}) at {_short(commit)}" + if commit else f"`{mr.RECORDS_DIR}/` of this repository") return "\n".join([ "# Measurements", "", @@ -148,8 +499,7 @@ def data_page_section(path: Path) -> list[str]: "## Measurements", "", "[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe " - "record at the pinned commit of [retrobios-measurements]" - "(https://github.com/Abdess/retrobios-measurements), and " + f"record kept in [`{mr.RECORDS_DIR}/`]({REPO}/tree/main/{mr.RECORDS_DIR}), and " "`downloads/measurements/.sqlite` the records of one emulator, linked " f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads " "[`data/measurements/index.json`](data/measurements/index.json). A record line is " @@ -160,7 +510,7 @@ def data_page_section(path: Path) -> list[str]: "|-------|---------|", *(f"| `{name}` | {', '.join(f'`{c}`' for c in columns)} |" for name, columns in sqlite_tables(path)), "", - "`meta` names the measurements commit and the record version. A list or object " + "`meta` names the commit the record links point to and the record version. A list or object " "value is JSON text, an absent one empty. `runs.superseded` is 1 for a record a " "later one corrects.", "", diff --git a/scripts/sync_measurements.py b/scripts/sync_measurements.py deleted file mode 100644 index f11a8d915..000000000 --- a/scripts/sync_measurements.py +++ /dev/null @@ -1,564 +0,0 @@ -#!/usr/bin/env python3 -"""Read the measurements repository at its pin and compile what the site shows. - -The records live in their own repository (Abdess/retrobios-measurements), -one append-only .jsonl per emulator. This repository holds only the pin, -`measurements-pin.json`: the repository URL and the commit read. The clone -is kept at .cache/measurements/ and read at the pinned commit with git, so -its working tree may hold records not yet committed. - -From the records at the pin it writes, never commits: - - public/data/measurements/index.json the compact index the explorer and - the emulator pages read, one item - per emulator and subject - public/downloads/measurements.sqlite every record, normalised - public/downloads/measurements/.sqlite one database per emulator - -`--fetch` clones or updates the clone, the only step that uses the network, -and refuses a remote whose history rewrote or removed a line the pin holds. -`--bump` then moves the pin to the fetched head. -""" - -from __future__ import annotations - -import argparse -import json -import os -import sqlite3 -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -import measurement_records as mr # noqa: E402 -from artifacts import json_text, write_if_changed, write_text_atomic # noqa: E402 - -ROOT = Path(__file__).resolve().parent.parent -PIN_FILE = ROOT / "measurements-pin.json" -CLONE = ROOT / ".cache" / "measurements" -INDEX_PATH = Path("data") / "measurements" / "index.json" -SQLITE_PATH = Path("downloads") / "measurements.sqlite" -PER_EMULATOR_DIR = Path("downloads") / "measurements" -SQLITE_SCHEMA_VERSION = "1" -INDEX_SCHEMA_VERSION = 1 - - -class MeasurementsError(RuntimeError): - """The measurements at the pin cannot be read, or break a rule.""" - - -@dataclass(frozen=True) -class Stored: - """One record line as read at a commit.""" - - emulator: str - line: int - record: dict - - -# -------------------------------------------------------------------------- -# Pin and git -# -------------------------------------------------------------------------- - -def load_pin(path: Path = PIN_FILE) -> dict: - pin = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(pin.get("repo"), str) or not isinstance(pin.get("commit"), str) \ - or len(pin["commit"]) != 40: - raise MeasurementsError(f"{path}: needs a repo URL and a 40-character commit") - return pin - - -def write_pin(pin: dict, path: Path = PIN_FILE) -> None: - write_text_atomic(str(path), json.dumps(pin, indent=2, sort_keys=True) + "\n") - - -def git(clone: Path, *args: str, binary: bool = False) -> str | bytes: - proc = subprocess.run(["git", "-C", str(clone), *args], capture_output=True, check=False) - if proc.returncode != 0: - raise MeasurementsError(f"git {' '.join(args)}: {proc.stderr.decode(errors='replace').strip()}") - return proc.stdout if binary else proc.stdout.decode("utf-8") - - -def has_commit(clone: Path, commit: str) -> bool: - if not (clone / ".git").exists(): - return False - proc = subprocess.run(["git", "-C", str(clone), "cat-file", "-e", f"{commit}^{{commit}}"], - capture_output=True, check=False) - return proc.returncode == 0 - - -def tree_files(clone: Path, commit: str) -> dict[str, str]: - """Path -> blob id of every file under records/ and data/ at a commit.""" - out: dict[str, str] = {} - listing = git(clone, "ls-tree", "-r", commit, "--", mr.RECORDS_DIR, mr.DATA_DIR) - for row in str(listing).splitlines(): - meta, _, path = row.partition("\t") - out[path] = meta.split()[2] - return out - - -def show(clone: Path, commit: str, path: str) -> bytes: - return bytes(git(clone, "show", f"{commit}:{path}", binary=True)) - - -# -------------------------------------------------------------------------- -# Reading and checking -# -------------------------------------------------------------------------- - -def read_records(clone: Path, commit: str) -> list[Stored]: - """Every record line at a commit, each checked against its rules and data.""" - files = tree_files(clone, commit) - problems: list[str] = [] - stored: list[Stored] = [] - for path in sorted(p for p in files if p.startswith(f"{mr.RECORDS_DIR}/")): - emulator = path[len(mr.RECORDS_DIR) + 1:].removesuffix(".jsonl") - if not path.endswith(".jsonl") or "/" in emulator: - problems.append(f"{path}: records/ holds .jsonl files only") - continue - seen: set[str] = set() - try: - records = mr.parse_lines(show(clone, commit, path).decode("utf-8"), path) - except mr.RecordError as exc: - problems.append(str(exc)) - continue - for number, record in enumerate(records, 1): - where = f"{path}:{number}" - problems.extend(f"{where}: {p}" for p in mr.check_line(record)) - if record.get("emulator") != emulator: - problems.append(f"{where}: emulator {record.get('emulator')!r} in {path}") - if record.get("id") in seen: - problems.append(f"{where}: id {record.get('id')} appears twice") - if record.get("supersedes") and record["supersedes"] not in seen: - problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file") - seen.add(record.get("id")) - data = mr.sidecar_path(emulator, str(record.get("id"))) - if data not in files: - problems.append(f"{where}: {data} is missing") - else: - problems.extend(f"{where}: {p}" for p in mr.check_pair(record, show(clone, commit, data))) - stored.append(Stored(emulator, number, record)) - if problems: - raise MeasurementsError("records at {}:\n {}".format(commit[:12], "\n ".join(problems))) - return stored - - -def append_only_problems(clone: Path, old: str, new: str) -> list[str]: - """What ``new`` changed or removed of the lines and data ``old`` holds.""" - before, after = tree_files(clone, old), tree_files(clone, new) - problems = [] - for path, blob in sorted(before.items()): - if path not in after: - problems.append(f"{path}: removed") - elif after[path] == blob: - continue - elif path.endswith(".jsonl"): - moved = mr.appended_only(show(clone, old, path).decode("utf-8"), - show(clone, new, path).decode("utf-8")) - if moved: - problems.append(f"{path}: {moved}") - else: - problems.append(f"{path}: rewritten") - return problems - - -# -------------------------------------------------------------------------- -# Index -# -------------------------------------------------------------------------- - -def host_summary(host: dict) -> str: - parts = [host.get("cpu_model") or "unknown CPU"] - if host.get("cpu_count"): - parts.append(f"{host['cpu_count']} CPUs") - parts.append(f"{host.get('machine', '')} Linux {host.get('kernel', '')}".strip() - if "Linux" in host.get("os", "") else host.get("os", "")) - parts.append(f"Python {host.get('python', '')}") - before, after = host.get("load_before"), host.get("load_after") - if before and after: - parts.append(f"load {before[0]} before, {after[0]} after") - return ", ".join(part for part in parts if part) - - -def bench_statement(record: dict) -> str: - timing = record["timing"] - frame, fps = timing["ms_per_frame"], timing["fps"] - return (f"median {frame['median']:.3f} ms per frame ({fps['median']:.1f} fps), " - f"min {frame['min']:.3f}, max {frame['max']:.3f}, over " - f"{record['inputs']['repetitions']} repetitions of {record['inputs']['frames']} frames") - - -def permalink(repo: str, commit: str, path: str, line: int | None = None) -> str: - return f"{repo}/blob/{commit}/{path}" + (f"#L{line}" if line else "") - - -def index_item(entry: Stored, runs: int, pin: dict, profiles: dict, - system_names: dict[str, str]) -> dict: - record = entry.record - profile = profiles.get(entry.emulator) or {} - check = record.get("check") or {} - core = record["core"] - content = (record["inputs"].get("content") or {}) - return { - "emulator": entry.emulator, - "emulator_name": str(profile.get("emulator", entry.emulator)), - "profiled": bool(profile), - "systems": sorted(set(profile.get("systems") or [])), - "system_names": [system_names.get(s) or s for s in sorted(set(profile.get("systems") or []))], - "subject": record["subject"], - "kind": record["kind"], - "claim": check.get("claim"), - "method": check.get("method"), - "observable": check.get("observable"), - "variable": check.get("variable"), - "status": (check.get("result") or {}).get("status") or ("measured" if record["kind"] == "bench" else "ran"), - "statement": (check.get("result") or {}).get("statement") - or (bench_statement(record) if record.get("timing") else ""), - "content": content.get("label"), - "variants": len(record["variants"]), - "repetitions": record["inputs"]["repetitions"], - "flags": len(record["flags"]), - "date": record["recorded_at"], - "host": host_summary(record["host"]), - "core": { - "library": " ".join(filter(None, [core.get("library_name"), core.get("library_version")])), - "file": core["file"], "sha256": core["sha256"], "repo": core.get("repo"), - "commit": core.get("commit"), "build": core.get("build_command"), - }, - "frontend_commit": record["frontend"].get("commit"), - "id": record["id"], - "runs": runs, - "record_url": permalink(pin["repo"], pin["commit"], mr.records_path(entry.emulator), entry.line), - "data_url": permalink(pin["repo"], pin["commit"], mr.sidecar_path(entry.emulator, record["id"])), - "sqlite": f"downloads/measurements/{entry.emulator}.sqlite", - } - - -def build_index(stored: list[Stored], pin: dict | None, profiles: dict, - system_names: dict[str, str] | None = None) -> dict: - """One item per (emulator, subject): its latest record not superseded.""" - by_subject: dict[tuple[str, str], list[Stored]] = {} - live_ids = {r["id"] for r in mr.live(s.record for s in stored)} - for entry in stored: - if entry.record["id"] in live_ids: - by_subject.setdefault((entry.emulator, entry.record["subject"]), []).append(entry) - items = [] - for key in sorted(by_subject): - entries = by_subject[key] - latest = max(entries, key=lambda s: (s.record["recorded_at"], s.line)) - items.append(index_item(latest, len(entries), pin or {}, profiles, system_names or {})) - items.sort(key=lambda item: (item["emulator"], item["date"], item["subject"])) - return { - "schema_version": INDEX_SCHEMA_VERSION, - "kind": "measurements", - "repo": (pin or {}).get("repo"), - "commit": (pin or {}).get("commit"), - "records": len(stored), - "count": len(items), - "items": items, - } - - -# -------------------------------------------------------------------------- -# SQLite -# -------------------------------------------------------------------------- - -SCHEMA_SQL = """ - PRAGMA journal_mode = OFF; - PRAGMA synchronous = OFF; - CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); - CREATE TABLE cores ( - sha256 TEXT PRIMARY KEY, file TEXT NOT NULL, repo TEXT, commit_sha TEXT, - build_command TEXT, compiler TEXT, library_name TEXT, library_version TEXT - ); - CREATE TABLE hosts ( - id INTEGER PRIMARY KEY, cpu_model TEXT, cpu_count INTEGER, os TEXT NOT NULL, - kernel TEXT NOT NULL, machine TEXT NOT NULL, python TEXT NOT NULL - ); - CREATE TABLE emulator_systems ( - emulator TEXT NOT NULL, system TEXT NOT NULL, PRIMARY KEY (emulator, system) - ); - CREATE TABLE runs ( - id TEXT PRIMARY KEY, emulator TEXT NOT NULL, subject TEXT NOT NULL, - kind TEXT NOT NULL, recorded_at TEXT NOT NULL, supersedes TEXT, - superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256), - host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT, - max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL, - script_modified INTEGER, flags TEXT NOT NULL, data_sha256 TEXT NOT NULL, - data_bytes INTEGER NOT NULL, record_url TEXT NOT NULL, line INTEGER NOT NULL, - migrated_from_version INTEGER - ); - CREATE TABLE inputs ( - run_id TEXT PRIMARY KEY REFERENCES runs(id), content_label TEXT, content_sha1 TEXT, - content_size INTEGER, options TEXT NOT NULL, env TEXT NOT NULL, frames INTEGER NOT NULL, - warmup INTEGER NOT NULL, repetitions INTEGER NOT NULL, presses TEXT NOT NULL, - observables TEXT NOT NULL, frame_hash_every INTEGER NOT NULL, joypad_ports INTEGER NOT NULL - ); - CREATE TABLE content_tracks ( - run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL, - file TEXT NOT NULL, sha1 TEXT NOT NULL, size INTEGER NOT NULL, - PRIMARY KEY (run_id, position) - ); - CREATE TABLE checks ( - run_id TEXT PRIMARY KEY REFERENCES runs(id), claim TEXT NOT NULL, method TEXT NOT NULL, - observable TEXT NOT NULL, variable TEXT NOT NULL, status TEXT NOT NULL, - statement TEXT NOT NULL - ); - CREATE TABLE check_groups ( - run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL, - value TEXT, value_sha1 TEXT, variants TEXT NOT NULL, PRIMARY KEY (run_id, position) - ); - CREATE TABLE variants ( - run_id TEXT NOT NULL REFERENCES runs(id), position INTEGER NOT NULL, - label TEXT NOT NULL, repetitions INTEGER NOT NULL, loaded INTEGER NOT NULL, - flags TEXT NOT NULL, observables TEXT NOT NULL, files_opened TEXT, - files_probed_absent TEXT, tracing TEXT, PRIMARY KEY (run_id, position) - ); - CREATE TABLE system_files ( - run_id TEXT NOT NULL REFERENCES runs(id), variant_position INTEGER NOT NULL, - destination TEXT NOT NULL, sha1 TEXT, - PRIMARY KEY (run_id, variant_position, destination) - ); - CREATE TABLE benchmarks ( - run_id TEXT NOT NULL REFERENCES runs(id), metric TEXT NOT NULL, - median REAL NOT NULL, mean REAL NOT NULL, stdev REAL NOT NULL, - min REAL NOT NULL, max REAL NOT NULL, PRIMARY KEY (run_id, metric) - ); - CREATE INDEX runs_emulator_idx ON runs(emulator, subject); - CREATE INDEX runs_subject_idx ON runs(subject); - CREATE INDEX emulator_systems_system_idx ON emulator_systems(system); - CREATE INDEX checks_claim_idx ON checks(claim); - CREATE INDEX system_files_sha1_idx ON system_files(sha1); -""" - -TABLES = ("meta", "cores", "hosts", "emulator_systems", "runs", "inputs", "content_tracks", - "checks", "check_groups", "variants", "system_files", "benchmarks") - - -def _host_key(host: dict) -> tuple: - return tuple(json_text(host.get(k)) for k in ("cpu_model", "cpu_count", "os", "kernel", - "machine", "python")) - - -def write_sqlite(destination: Path, stored: list[Stored], pin: dict | None, profiles: dict, - scratch: Path) -> None: - """A deterministic database of the records: sorted rows, no clock.""" - scratch.mkdir(parents=True, exist_ok=True) - temp = scratch / f"{destination.stem}-{os.getpid()}.sqlite" - temp.unlink(missing_ok=True) - destination.parent.mkdir(parents=True, exist_ok=True) - stored = sorted(stored, key=lambda s: (s.emulator, s.line)) - live_ids = {r["id"] for r in mr.live(s.record for s in stored)} - hosts = {key: number for number, key in enumerate(sorted({_host_key(s.record["host"]) for s in stored}), 1)} - connection = sqlite3.connect(temp) - try: - connection.executescript(SCHEMA_SQL) - meta = { - "schema_version": SQLITE_SCHEMA_VERSION, - "record_version": str(mr.RECORD_VERSION), - "measurements_repo": (pin or {}).get("repo") or "", - "measurements_commit": (pin or {}).get("commit") or "", - "license": "MIT", - "scope": "core probe records: the record lines, not their data files", - } - connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items())) - cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored} - connection.executemany("INSERT INTO cores VALUES (?, ?, ?, ?, ?, ?, ?, ?)", [ - (sha, core["file"], json_text(core.get("repo")), json_text(core.get("commit")), - json_text(core.get("build_command")), json_text(core.get("compiler")), - json_text(core.get("library_name")), json_text(core.get("library_version"))) - for sha, core in sorted(cores.items())]) - connection.executemany("INSERT INTO hosts VALUES (?, ?, ?, ?, ?, ?, ?)", [ - (number, *key) for key, number in sorted(hosts.items(), key=lambda kv: kv[1])]) - emulators = sorted({s.emulator for s in stored}) - connection.executemany("INSERT INTO emulator_systems VALUES (?, ?)", [ - (emu, system) for emu in emulators - for system in sorted(set((profiles.get(emu) or {}).get("systems") or []))]) - for entry in stored: - _insert_run(connection, entry, pin or {}, hosts, entry.record["id"] not in live_ids) - connection.commit() - connection.execute("VACUUM") - finally: - connection.close() - os.replace(temp, destination) - - -def _insert_run(connection: sqlite3.Connection, entry: Stored, pin: dict, - hosts: dict, superseded: bool) -> None: - r = entry.record - host, front, inputs = r["host"], r["frontend"], r["inputs"] - rid = r["id"] - connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( - rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"), - int(superseded), r["core"]["sha256"], hosts[_host_key(host)], - json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"), - front.get("commit"), front["script_sha256"], - None if front.get("script_modified") is None else int(front["script_modified"]), - json_text(r["flags"]), r["data"]["sha256"], r["data"]["bytes"], - permalink(pin.get("repo", ""), pin.get("commit", ""), mr.records_path(entry.emulator), entry.line), - entry.line, r.get("migrated_from_version"))) - content = inputs.get("content") or {} - connection.execute("INSERT INTO inputs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( - rid, content.get("label"), content.get("sha1"), content.get("size"), - json_text(inputs["options"]), json_text(inputs["env"]), inputs["frames"], inputs["warmup"], - inputs["repetitions"], json_text(inputs["presses"]), json_text(inputs["observables"]), - inputs["frame_hash_every"], inputs["joypad_ports"])) - connection.executemany("INSERT INTO content_tracks VALUES (?, ?, ?, ?, ?)", [ - (rid, position, track["file"], track["sha1"], track["size"]) - for position, track in enumerate(content.get("tracks") or [])]) - check = r.get("check") - if check: - result = check["result"] - connection.execute("INSERT INTO checks VALUES (?, ?, ?, ?, ?, ?, ?)", ( - rid, check["claim"], check["method"], check["observable"], check["variable"], - result["status"], result["statement"])) - connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [ - (rid, position, json_text(group["value"]) if "value" in group else None, - group.get("value_sha1"), json_text(group["variants"])) - for position, group in enumerate(result["groups"])]) - for position, variant in enumerate(r["variants"]): - connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( - rid, position, variant["label"], variant["repetitions"], int(variant["loaded"]), - json_text(variant["flags"]), json_text(variant["observables"]), - json_text(variant["files_opened"]), json_text(variant["files_probed_absent"]), - json_text(variant["tracing"]))) - connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [ - (rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())]) - for metric, stats in sorted((r.get("timing") or {}).items()): - connection.execute("INSERT INTO benchmarks VALUES (?, ?, ?, ?, ?, ?, ?)", ( - rid, metric, stats["median"], stats["mean"], stats["stdev"], stats["min"], stats["max"])) - - -# -------------------------------------------------------------------------- -# Building the site's files -# -------------------------------------------------------------------------- - -@dataclass -class Built: - index: dict - paths: list[Path] - - -def load_profiles(emulators_dir: Path) -> dict: - from common import load_emulator_profiles # noqa: PLC0415 - - return load_emulator_profiles(str(emulators_dir), skip_aliases=False) - - -def records_at_pin(clone: Path, pin: dict, require: bool) -> list[Stored] | None: - """The records at the pin, or None when the clone does not hold it.""" - if has_commit(clone, pin["commit"]): - return read_records(clone, pin["commit"]) - message = (f"measurements commit {pin['commit'][:12]} is not in {clone}: run " - f"python scripts/sync_measurements.py --fetch") - if require: - raise MeasurementsError(message) - print(f"WARNING: {message}. The site lists no measurements.", file=sys.stderr) - return None - - -def build(public: Path, clone: Path = CLONE, pin_path: Path = PIN_FILE, - emulators_dir: Path = ROOT / "emulators", require: bool = False, - scratch: Path = ROOT / "tmp" / "site", - system_names: dict[str, str] | None = None) -> Built: - """Write the index and the databases under the site's public folder.""" - pin = load_pin(pin_path) - stored = records_at_pin(clone, pin, require) - profiles = load_profiles(emulators_dir) - if stored is None: - pin = None - index = build_index(stored or [], pin, profiles, system_names) - paths = [public / INDEX_PATH] - (public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True) - write_if_changed(str(public / INDEX_PATH), - json.dumps(index, ensure_ascii=False, indent=1, sort_keys=True) + "\n") - write_sqlite(public / SQLITE_PATH, stored or [], pin, profiles, scratch) - paths.append(public / SQLITE_PATH) - for emulator in sorted({s.emulator for s in stored or []}): - target = public / PER_EMULATOR_DIR / f"{emulator}.sqlite" - write_sqlite(target, [s for s in stored or [] if s.emulator == emulator], pin, profiles, scratch) - paths.append(target) - return Built(index, paths) - - -def fetch(clone: Path, pin: dict) -> str: - """Clone or update the measurements repository. Returns the remote head.""" - if not (clone / ".git").exists(): - clone.parent.mkdir(parents=True, exist_ok=True) - proc = subprocess.run(["git", "clone", "--quiet", pin["repo"], str(clone)], - capture_output=True, text=True, check=False) - if proc.returncode != 0: - raise MeasurementsError(f"git clone {pin['repo']}: {proc.stderr.strip()}") - else: - git(clone, "fetch", "--quiet", "origin") - head = str(git(clone, "rev-parse", "refs/remotes/origin/HEAD")).strip() - branch = str(git(clone, "rev-parse", "--abbrev-ref", "HEAD")).strip() - if branch != "HEAD": - proc = subprocess.run(["git", "-C", str(clone), "merge", "--ff-only", "--quiet", head], - capture_output=True, text=True, check=False) - if proc.returncode != 0: - print(f"WARNING: {clone} has local commits, its branch was left as is: " - f"{proc.stderr.strip()}", file=sys.stderr) - return head - - -def remote_head(repo: str) -> str: - proc = subprocess.run(["git", "ls-remote", repo, "HEAD"], capture_output=True, text=True, - check=False, timeout=60) - if proc.returncode != 0 or not proc.stdout.strip(): - raise MeasurementsError(f"git ls-remote {repo}: {proc.stderr.strip() or 'no HEAD'}") - return proc.stdout.split()[0] - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__.split("\n\n")[0]) - parser.add_argument("--fetch", action="store_true", - help="clone or update the clone, and check the remote kept every line the pin holds") - parser.add_argument("--bump", action="store_true", - help=f"after --fetch, move {PIN_FILE.name} to the fetched head") - parser.add_argument("--check-only", action="store_true", - help="check the records at the pin and write nothing") - parser.add_argument("--clone", default=str(CLONE), help="the clone (default: %(default)s)") - parser.add_argument("--pin", default=str(PIN_FILE), help="the pin file (default: %(default)s)") - parser.add_argument("--site-dir", default=str(ROOT / "website"), - help="the site project whose public/ folder is written (default: %(default)s)") - args = parser.parse_args(argv) - if args.bump and not args.fetch: - parser.error("--bump needs --fetch") - clone, pin_path = Path(args.clone), Path(args.pin) - try: - pin = load_pin(pin_path) - if args.fetch: - head = fetch(clone, pin) - problems = append_only_problems(clone, pin["commit"], head) \ - if has_commit(clone, pin["commit"]) else [f"pin {pin['commit']} is not in the remote history"] - if problems: - print("ERROR: the remote changed records the pin holds:\n " + "\n ".join(problems), - file=sys.stderr) - return 1 - records = read_records(clone, head) - pinned = len(read_records(clone, pin["commit"])) - print(f"{head[:12]}: {len(records)} records, {len(records) - pinned} after the pin " - f"{pin['commit'][:12]}") - if args.bump and head != pin["commit"]: - write_pin({**pin, "commit": head}, pin_path) - print(f"pin moved to {head}") - return 0 - if args.check_only: - stored = records_at_pin(clone, pin, require=True) - print(f"{pin['commit'][:12]}: {len(stored or [])} records, every line and data file checked") - return 0 - built = build(Path(args.site_dir) / "public", clone, pin_path, require=True) - print(f"{built.index['records']} records, {built.index['count']} subjects: " - + ", ".join(str(p.relative_to(Path(args.site_dir))) for p in built.paths)) - return 0 - except MeasurementsError as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - from common import run_cli - - run_cli(main) diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index b34cf582d..683a4565b 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -593,46 +593,33 @@ def _format_errors() -> list[str]: ] -def _measurement_errors(clone: Path | None = None, pin_path: Path | None = None) -> list[str]: - """Every record line and data file at the measurements pin, against their - schemas and the rules sync_measurements.py applies. - - The records live in their own repository. Without a clone holding the - pin there is nothing to read, and the run says so. - """ - import sync_measurements # noqa: PLC0415 +def _measurement_errors(root: Path | None = None) -> list[str]: + """Every record line and data file under measurements/, against their + schemas and the rules measurement_records.read_records applies.""" import measurement_records as mr # noqa: PLC0415 - clone = clone or sync_measurements.CLONE - pin = sync_measurements.load_pin(pin_path or sync_measurements.PIN_FILE) - if not sync_measurements.has_commit(clone, pin["commit"]): - print(f"SKIP measurements: {pin['commit'][:12]} is not in {clone} " - "(python scripts/sync_measurements.py --fetch)") - return [] - commit = pin["commit"] + root = root or ROOT record_validator = _validator("core-probe-record.schema.json") data_validator = _validator("core-probe-data.schema.json") out: list[str] = [] - files = sync_measurements.tree_files(clone, commit) + files = mr.tree_files(root) for path in sorted(p for p in files if p.endswith(".jsonl")): - text = sync_measurements.show(clone, commit, path).decode("utf-8") - for number, line in enumerate(text.splitlines(), 1): + emulator = path[len(mr.RECORDS_DIR) + 1:].removesuffix(".jsonl") + for number, line in enumerate(files[path].read_text(encoding="utf-8").splitlines(), 1): try: record = json.loads(line) except json.JSONDecodeError as exc: - out.append(f"measurements {path}:{number}: {exc}") + out.append(f"{path}:{number}: {exc}") continue - out.extend(_errors(record_validator, record, f"measurements {path}:{number}")) - data = mr.sidecar_path(path[len("records/"):-len(".jsonl")], str(record.get("id"))) + out.extend(_errors(record_validator, record, f"{path}:{number}")) + data = mr.sidecar_path(emulator, str(record.get("id"))) if data in files: - out.extend(_errors(data_validator, - mr.read_sidecar(sync_measurements.show(clone, commit, data)), - f"measurements {data}")) + out.extend(_errors(data_validator, mr.read_sidecar(files[data].read_bytes()), data)) if not out: try: - sync_measurements.read_records(clone, commit) - except sync_measurements.MeasurementsError as exc: - out.append(f"measurements {exc}") + mr.read_records(root) + except mr.MeasurementsError as exc: + out.append(str(exc)) return out diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py index 0957ff455..062397f96 100644 --- a/tests/test_core_probe.py +++ b/tests/test_core_probe.py @@ -263,9 +263,10 @@ class FakeCoreTests(unittest.TestCase): self.assertEqual(record["emulator"], "fake") self.assertEqual(sorted(p.relative_to(record_dir).as_posix() for p in record_dir.rglob("*") if p.is_file()), - sorted(["records/fake.jsonl", f"data/fake/{self._only_id(record_dir)}.json.gz"])) - line = json.loads((record_dir / "records" / "fake.jsonl").read_text()) - blob = (record_dir / "data" / "fake" / f"{line['id']}.json.gz").read_bytes() + sorted(["measurements/fake.jsonl", + f"measurements/data/fake/{self._only_id(record_dir)}.json.gz"])) + line = json.loads((record_dir / "measurements" / "fake.jsonl").read_text()) + blob = (record_dir / "measurements" / "data" / "fake" / f"{line['id']}.json.gz").read_bytes() self.assertEqual(measurement_records.check_line(line), []) self.assertEqual(measurement_records.check_pair(line, blob), []) # The line and its sidecar give back the record the run printed. @@ -281,14 +282,14 @@ class FakeCoreTests(unittest.TestCase): @staticmethod def _only_id(record_dir: Path) -> str: - return json.loads((record_dir / "records" / "fake.jsonl").read_text())["id"] + return json.loads((record_dir / "measurements" / "fake.jsonl").read_text())["id"] def test_second_run_is_appended_beside_the_first(self) -> None: record_dir = self.tmp / "appended" argv = ["run", "--core", str(self.core), "--frames", "1", "--no-trace", "--max-load", "1000", "--record", str(record_dir), "--subject", "smoke"] self.assertEqual(self.probe(argv)[0], 0) - lines = record_dir / "records" / "fake.jsonl" + lines = record_dir / "measurements" / "fake.jsonl" first = lines.read_text() code, record = self.probe(argv) self.assertEqual(code, 0) @@ -301,16 +302,11 @@ class FakeCoreTests(unittest.TestCase): self.assertEqual(code, 2) self.assertEqual(len(lines.read_text().splitlines()), 3) - def test_recording_inside_the_tracked_tree_is_refused(self) -> None: - inside = REPO_ROOT / "measurements" - err = io.StringIO() - with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): - code = core_probe.main(["run", "--core", str(self.core), "--frames", "1", "--no-trace", - "--record", str(inside), "--work-dir", str(self.tmp / "work")]) - self.assertEqual(code, 2) - self.assertIn("inside the tracked tree", err.getvalue()) - self.assertIn(".cache/measurements", err.getvalue()) - self.assertFalse(inside.exists()) + def test_record_defaults_to_this_checkout(self) -> None: + parser = core_probe.build_parser() + args = parser.parse_args(["run", "--core", sys.executable, "--record", "--frames", "1"]) + self.assertEqual(Path(args.record), REPO_ROOT) + self.assertIsNone(parser.parse_args(["run", "--core", sys.executable]).record) def test_bench_stats(self) -> None: code, record = self.probe(["bench", "--core", str(self.core), "--frames", "2", "--warmup", "1", diff --git a/tests/test_measurements.py b/tests/test_measurements.py index 9b3d5c41e..8f4a96927 100644 --- a/tests/test_measurements.py +++ b/tests/test_measurements.py @@ -1,16 +1,15 @@ -"""Core probe records in their own repository, and what the site makes of them. +"""Core probe records under measurements/, and what the site makes of them. -A synthetic measurements repository is built with git under tmp/tests: -records split into a line and a sidecar, read at a pinned commit, compiled -into the index and the SQLite files. The append-only rule is checked the -way --fetch applies it, and two branches that each add a line merge. +Synthetic record trees are built under tmp/tests: records split into a line +and a sidecar, read from the tree, compiled into the index and the SQLite +files. The append-only rule is checked against a commit, two branches that +each add a line merge, and the records this repository holds are checked +against HEAD. """ from __future__ import annotations -import contextlib import copy -import io import json import os import re @@ -26,10 +25,10 @@ REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT / "scripts")) import measurement_records as mr # noqa: E402 -import sitemeasurements # noqa: E402 -import sync_measurements as sm # noqa: E402 +import sitemeasurements as sm # noqa: E402 -REPO_URL = "https://github.com/Abdess/retrobios-measurements" +REPO_URL = "https://github.com/Abdess/retrobios" +COMMIT = "e" * 40 SHA1 = "a" * 40 SHA256 = "b" * 64 @@ -95,6 +94,8 @@ def full_record(emulator: str, subject: str, when: str, kind: str = "check", return record + + def git(cwd: Path, *args: str) -> str: env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", "GIT_COMMITTER_EMAIL": "t@t", "GIT_CEILING_DIRECTORIES": str(cwd.parent)} @@ -122,32 +123,33 @@ def commit_all(tree: Path, message: str) -> str: class Workspace(unittest.TestCase): - """A measurements repository with three records and a pin on it.""" + """A tree with three records under measurements/ and one profile.""" def setUp(self) -> None: (REPO_ROOT / "tmp" / "tests").mkdir(parents=True, exist_ok=True) self.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests")) self.addCleanup(shutil.rmtree, self.tmp, True) - self.repo = self.tmp / "remote" - self.repo.mkdir() - git(self.repo, "init", "-q", "-b", "main") - (self.repo / ".gitattributes").write_text("records/*.jsonl merge=union\n") - self.first = add_record(self.repo, full_record("alpha", "card", "2026-10-01T10:00:00+00:00")) - self.bench = add_record(self.repo, full_record("alpha", "speed", "2026-10-01T11:00:00+00:00", + self.tree = self.tmp / "tree" + self.tree.mkdir() + self.first = add_record(self.tree, full_record("alpha", "card", "2026-10-01T10:00:00+00:00")) + self.bench = add_record(self.tree, full_record("alpha", "speed", "2026-10-01T11:00:00+00:00", kind="bench")) - self.beta = add_record(self.repo, full_record("beta", "card", "2026-10-02T10:00:00+00:00", + self.beta = add_record(self.tree, full_record("beta", "card", "2026-10-02T10:00:00+00:00", values=("1" * 40, "1" * 40))) - self.commit = commit_all(self.repo, "feat: first records") - self.pin = self.tmp / "pin.json" - sm.write_pin({"repo": str(self.repo), "commit": self.commit}, self.pin) self.emulators = self.tmp / "emulators" self.emulators.mkdir() (self.emulators / "alpha.yml").write_text( 'emulator: "Alpha"\ntype: libretro\nsystems:\n - sega-saturn\nfiles: []\n') - def built(self, public: Path, clone: Path | None = None) -> sm.Built: - return sm.build(public, clone or self.repo, self.pin, self.emulators, require=True, - scratch=self.tmp / "scratch", system_names={"sega-saturn": "Sega Saturn"}) + def built(self, public: Path) -> sm.Built: + return sm.build(public, self.tree, self.emulators, scratch=self.tmp / "scratch", + system_names={"sega-saturn": "Sega Saturn"}, repo=REPO_URL, commit=COMMIT) + + def repository(self) -> str: + """Make the tree a git repository, its records committed on main.""" + git(self.tree, "init", "-q", "-b", "main") + (self.tree / ".gitattributes").write_text("measurements/*.jsonl merge=union\n") + return commit_all(self.tree, "feat: first records") class RecordSplit(unittest.TestCase): @@ -211,17 +213,29 @@ class RecordSplit(unittest.TestCase): self.assertTrue(mr.check_pair(forged, blob)) -class SyncAndIndex(Workspace): - def test_records_are_read_at_the_pin_not_the_working_tree(self) -> None: - add_record(self.repo, full_record("alpha", "card", "2026-10-03T10:00:00+00:00")) - records = sm.read_records(self.repo, self.commit) - self.assertEqual(len(records), 3) + + +class ReadAndIndex(Workspace): + def test_records_are_read_from_the_tree(self) -> None: + records = mr.read_records(self.tree) + self.assertEqual([(s.emulator, s.line) for s in records], [("alpha", 1), ("alpha", 2), ("beta", 1)]) + self.assertEqual(mr.read_records(self.tmp / "empty"), []) + + def test_a_broken_tree_is_named(self) -> None: + stray = self.tree / mr.DATA_DIR / "beta" / "2026-10-02-000000000000.json.gz" + stray.write_bytes(b"x") + (self.tree / mr.sidecar_path("alpha", self.bench["id"])).unlink() + with self.assertRaises(mr.MeasurementsError) as caught: + mr.read_records(self.tree) + message = str(caught.exception) + self.assertIn(f"{mr.sidecar_path('alpha', self.bench['id'])} is missing", message) + self.assertIn(f"{mr.DATA_DIR}/beta/2026-10-02-000000000000.json.gz: no record line names it", message) def test_index_shape(self) -> None: index = self.built(self.tmp / "public").index self.assertEqual({k: index[k] for k in ("schema_version", "kind", "repo", "commit", "records", "count")}, - {"schema_version": 1, "kind": "measurements", "repo": str(self.repo), - "commit": self.commit, "records": 3, "count": 3}) + {"schema_version": 1, "kind": "measurements", "repo": REPO_URL, + "commit": COMMIT, "records": 3, "count": 3}) keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind", "claim", "method", "observable", "variable", "status", "statement", "content", "variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id", @@ -232,18 +246,28 @@ class SyncAndIndex(Workspace): self.assertEqual(alpha["card"]["status"], "differs") self.assertEqual(alpha["card"]["system_names"], ["Sega Saturn"]) self.assertEqual(alpha["card"]["record_url"], - f"{self.repo}/blob/{self.commit}/records/alpha.jsonl#L1") + f"{REPO_URL}/blob/{COMMIT}/measurements/alpha.jsonl#L1") + self.assertEqual(alpha["card"]["data_url"], + f"{REPO_URL}/blob/{COMMIT}/measurements/data/alpha/{self.first['id']}.json.gz") self.assertEqual(alpha["speed"]["status"], "measured") self.assertIn("ms per frame", alpha["speed"]["statement"]) self.assertIn("Test CPU", alpha["speed"]["host"]) written = json.loads((self.tmp / "public" / sm.INDEX_PATH).read_text()) self.assertEqual(written, index) + def test_links_name_the_last_commit_that_touched_the_records(self) -> None: + first = self.repository() + (self.tree / "other.txt").write_text("x") + commit_all(self.tree, "chore: unrelated") + self.assertEqual(mr.last_commit(self.tree), first) + index = sm.build(self.tmp / "public", self.tree, self.emulators, scratch=self.tmp / "scratch").index + self.assertEqual(index["commit"], first) + self.assertTrue(all(f"/blob/{first}/measurements/" in i["record_url"] for i in index["items"])) + def test_latest_record_of_a_subject_wins_and_a_superseded_one_is_dropped(self) -> None: later = full_record("alpha", "card", "2026-10-05T10:00:00+00:00", values=("3" * 40, "3" * 40)) later["supersedes"] = self.first["id"] - add_record(self.repo, later) - sm.write_pin({"repo": str(self.repo), "commit": commit_all(self.repo, "feat: rerun")}, self.pin) + add_record(self.tree, later) card = next(i for i in self.built(self.tmp / "public").index["items"] if i["emulator"] == "alpha" and i["subject"] == "card") self.assertEqual((card["status"], card["runs"]), ("same", 1)) @@ -257,7 +281,7 @@ class SyncAndIndex(Workspace): "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")] self.assertEqual(tuple(tables), sm.TABLES) count = lambda table: connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: E731 - records = [s.record for s in sm.read_records(self.repo, self.commit)] + records = [s.record for s in mr.read_records(self.tree)] self.assertEqual(count("runs"), len(records)) self.assertEqual(count("checks"), sum(1 for r in records if r.get("check"))) self.assertEqual(count("benchmarks"), 2 * sum(1 for r in records if r.get("timing"))) @@ -266,10 +290,12 @@ class SyncAndIndex(Workspace): sum(len(r["check"]["result"]["groups"]) for r in records if r.get("check"))) self.assertEqual(count("hosts"), 1) self.assertEqual(count("cores"), 1) - self.assertEqual(dict(connection.execute("SELECT key, value FROM meta"))["measurements_commit"], - self.commit) + meta = dict(connection.execute("SELECT key, value FROM meta")) + self.assertEqual((meta["measurements_repo"], meta["measurements_commit"]), (REPO_URL, COMMIT)) self.assertEqual(connection.execute("SELECT system FROM emulator_systems").fetchall(), [("sega-saturn",)]) + self.assertEqual(connection.execute("SELECT record_url FROM runs WHERE line = 1 AND emulator = 'beta'") + .fetchone()[0], f"{REPO_URL}/blob/{COMMIT}/measurements/beta.jsonl#L1") indexes = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'index'")} self.assertTrue({"runs_emulator_idx", "emulator_systems_system_idx", "checks_claim_idx"} <= indexes) connection.close() @@ -286,15 +312,11 @@ class SyncAndIndex(Workspace): for path in first.paths: self.assertEqual(path.read_bytes(), (two / path.relative_to(one)).read_bytes(), path.name) - def test_a_missing_pin_fails_only_when_required(self) -> None: - sm.write_pin({"repo": str(self.repo), "commit": "f" * 40}, self.pin) - with self.assertRaises(sm.MeasurementsError): - self.built(self.tmp / "public") - with contextlib.redirect_stderr(io.StringIO()) as err: - built = sm.build(self.tmp / "public", self.repo, self.pin, self.emulators, require=False, - scratch=self.tmp / "scratch") - self.assertIn("WARNING", err.getvalue()) - self.assertEqual((built.index["count"], built.index["commit"]), (0, None)) + def test_no_records_writes_an_empty_index(self) -> None: + shutil.rmtree(self.tree / mr.RECORDS_DIR) + built = self.built(self.tmp / "public") + self.assertEqual((built.index["records"], built.index["count"]), (0, 0)) + self.assertEqual(len(built.paths), 2) def test_every_line_validates_against_the_schema(self) -> None: try: @@ -303,68 +325,82 @@ class SyncAndIndex(Workspace): self.skipTest("jsonschema is not installed") import validate_schemas # noqa: PLC0415 - self.assertEqual(validate_schemas._measurement_errors(self.repo, self.pin), []) - lines = self.repo / mr.records_path("beta") + self.assertEqual(validate_schemas._measurement_errors(self.tree), []) + lines = self.tree / mr.records_path("beta") record = json.loads(lines.read_text()) record["kind"] = "guess" lines.write_text(mr.line_text(record) + "\n") - sm.write_pin({"repo": str(self.repo), "commit": commit_all(self.repo, "bad")}, self.pin) - errors = validate_schemas._measurement_errors(self.repo, self.pin) - self.assertTrue(any("records/beta.jsonl:1" in e and "kind" in e for e in errors), errors) + errors = validate_schemas._measurement_errors(self.tree) + self.assertTrue(any("measurements/beta.jsonl:1" in e and "kind" in e for e in errors), errors) self.assertIsNotNone(Draft202012Validator) class AppendOnly(Workspace): - def fetch(self, *extra: str) -> tuple[int, str]: - err = io.StringIO() - with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): - code = sm.main(["--fetch", "--clone", str(self.tmp / "clone"), "--pin", str(self.pin), *extra]) - return code, err.getvalue() + def setUp(self) -> None: + super().setUp() + self.commit = self.repository() - def test_an_appended_line_moves_the_pin(self) -> None: - self.assertEqual(self.fetch()[0], 0) - add_record(self.repo, full_record("alpha", "card", "2026-10-04T10:00:00+00:00")) - head = commit_all(self.repo, "feat: rerun") - self.assertEqual(self.fetch("--bump")[0], 0) - self.assertEqual(sm.load_pin(self.pin)["commit"], head) + def test_an_appended_line_and_a_new_file_pass(self) -> None: + add_record(self.tree, full_record("alpha", "card", "2026-10-04T10:00:00+00:00")) + add_record(self.tree, full_record("gamma", "card", "2026-10-04T10:00:00+00:00")) + self.assertEqual(mr.append_only_problems(self.tree, "HEAD"), []) - def test_a_changed_line_is_refused(self) -> None: - lines = self.repo / mr.records_path("alpha") + def test_a_changed_line_is_named(self) -> None: + lines = self.tree / mr.records_path("alpha") text = lines.read_text().splitlines() text[0] = text[0].replace('"Each card, twice."', '"Each card, once."') lines.write_text("\n".join(text) + "\n") - commit_all(self.repo, "rewrite") - code, err = self.fetch("--bump") - self.assertEqual(code, 1) - self.assertIn("records/alpha.jsonl: line 1 changed", err) - self.assertEqual(sm.load_pin(self.pin)["commit"], self.commit) + self.assertEqual(mr.append_only_problems(self.tree, "HEAD"), + ["measurements/alpha.jsonl: line 1 changed"]) - def test_a_removed_line_or_data_file_is_refused(self) -> None: - lines = self.repo / mr.records_path("alpha") + def test_a_removed_line_or_data_file_is_named(self) -> None: + lines = self.tree / mr.records_path("alpha") lines.write_text(lines.read_text().splitlines()[0] + "\n") - (self.repo / mr.sidecar_path("beta", self.beta["id"])).unlink() - commit_all(self.repo, "remove") - code, err = self.fetch() - self.assertEqual(code, 1) - self.assertIn("records/alpha.jsonl: line 2 removed", err) - self.assertIn(f"{mr.sidecar_path('beta', self.beta['id'])}: removed", err) + (self.tree / mr.sidecar_path("beta", self.beta["id"])).unlink() + problems = mr.append_only_problems(self.tree, "HEAD") + self.assertIn("measurements/alpha.jsonl: line 2 removed", problems) + self.assertIn(f"{mr.sidecar_path('beta', self.beta['id'])}: removed", problems) + + def test_a_rewritten_data_file_is_named(self) -> None: + data = self.tree / mr.sidecar_path("beta", self.beta["id"]) + data.write_bytes(data.read_bytes() + b"x") + self.assertEqual(mr.append_only_problems(self.tree, "HEAD"), + [f"{mr.sidecar_path('beta', self.beta['id'])}: rewritten"]) def test_two_branches_adding_a_line_merge_cleanly(self) -> None: - git(self.repo, "checkout", "-q", "-b", "left") - add_record(self.repo, full_record("alpha", "card", "2026-10-06T10:00:00+00:00")) - commit_all(self.repo, "left") - git(self.repo, "checkout", "-q", "main") - git(self.repo, "checkout", "-q", "-b", "right") - add_record(self.repo, full_record("alpha", "speed", "2026-10-06T11:00:00+00:00", kind="bench", + git(self.tree, "checkout", "-q", "-b", "left") + add_record(self.tree, full_record("alpha", "card", "2026-10-06T10:00:00+00:00")) + commit_all(self.tree, "left") + git(self.tree, "checkout", "-q", "main") + git(self.tree, "checkout", "-q", "-b", "right") + add_record(self.tree, full_record("alpha", "speed", "2026-10-06T11:00:00+00:00", kind="bench", values=("4" * 40,))) - commit_all(self.repo, "right") - git(self.repo, "merge", "-q", "--no-edit", "left") - merged = git(self.repo, "rev-parse", "HEAD") - self.assertEqual(sm.append_only_problems(self.repo, self.commit, merged), []) - records = sm.read_records(self.repo, merged) + commit_all(self.tree, "right") + git(self.tree, "merge", "-q", "--no-edit", "left") + self.assertEqual(mr.append_only_problems(self.tree, self.commit), []) + records = mr.read_records(self.tree) self.assertEqual(sum(1 for s in records if s.emulator == "alpha"), 4) +class CommittedRecords(unittest.TestCase): + """The records this repository holds.""" + + def test_no_line_or_data_file_changed_or_vanished_since_head(self) -> None: + if subprocess.run(["git", "-C", str(REPO_ROOT), "rev-parse", "--verify", "-q", "HEAD"], + capture_output=True, check=False).returncode != 0: + self.skipTest("not a git checkout") + self.assertEqual(mr.append_only_problems(REPO_ROOT, "HEAD"), []) + + def test_every_record_reads_and_joins_its_data(self) -> None: + records = mr.read_records(REPO_ROOT) + self.assertGreaterEqual(len(records), 7) + + def test_jsonl_files_merge_by_union(self) -> None: + rules = (REPO_ROOT / ".gitattributes").read_text().splitlines() + rule = next(line for line in rules if line.startswith(f"{mr.RECORDS_DIR}/*.jsonl ")) + self.assertIn("merge=union", rule.split()) + + class SitePages(unittest.TestCase): ITEM = { "emulator": "beetle_saturn", "emulator_name": "Beetle Saturn", "profiled": True, @@ -376,14 +412,14 @@ class SitePages(unittest.TestCase): "host": "Test CPU, 12 CPUs", "core": {"library": "Beetle Saturn v1.32.1", "commit": "2ce77e2" * 5 + "abcde", "file": "x.so", "sha256": SHA256, "repo": None, "build": "make"}, "frontend_commit": None, "id": "2026-10-27-43527b763ace", "runs": 2, - "record_url": f"{REPO_URL}/blob/{'e' * 40}/records/beetle_saturn.jsonl#L4", - "data_url": f"{REPO_URL}/blob/{'e' * 40}/data/beetle_saturn/2026-10-27-43527b763ace.json.gz", + "record_url": f"{REPO_URL}/blob/{'e' * 40}/measurements/beetle_saturn.jsonl#L4", + "data_url": f"{REPO_URL}/blob/{'e' * 40}/measurements/data/beetle_saturn/2026-10-27-43527b763ace.json.gz", "sqlite": "downloads/measurements/beetle_saturn.sqlite", } def test_tested_block(self) -> None: - self.assertEqual(sitemeasurements.tested_section("beetle_saturn", []), []) - text = "\n".join(sitemeasurements.tested_section("beetle_saturn", [self.ITEM])) + self.assertEqual(sm.tested_section("beetle_saturn", []), []) + text = "\n".join(sm.tested_section("beetle_saturn", [self.ITEM])) self.assertTrue(text.startswith("## Tested\n")) for expected in ("**Result:** `same`", "**Method:** Twice per card. Every input fixed.", "**Ran on:** Test CPU, 12 CPUs", self.ITEM["record_url"], self.ITEM["data_url"], "1 flag raised", "2 runs", @@ -392,10 +428,10 @@ class SitePages(unittest.TestCase): self.assertIn(expected, text) def test_hosts_table_on_the_method_page(self) -> None: - page = sitemeasurements.with_hosts(f"x\n\n{sitemeasurements.HOSTS_MARKER}\n", + page = sm.with_hosts(f"x\n\n{sm.HOSTS_MARKER}\n", {"items": [self.ITEM, {**self.ITEM, "host": "Test CPU, 12 CPUs, load 1 before, 2 after"}]}) self.assertIn("| Test CPU, 12 CPUs | 2 |", page) - self.assertNotIn(sitemeasurements.HOSTS_MARKER, page) + self.assertNotIn(sm.HOSTS_MARKER, page) def test_the_explorer_reads_the_index(self) -> None: source = (REPO_ROOT / "website" / "src" / "components" / "MeasurementsExplorer.astro").read_text() @@ -404,37 +440,44 @@ class SitePages(unittest.TestCase): read = set(re.findall(r"\b(?:item|i)\.([a-z_]+)", source)) - {"length"} self.assertTrue(read) self.assertLessEqual(read, set(self.ITEM), read - set(self.ITEM)) - page = sitemeasurements.generate_explorer_page({"records": 7, "count": 7, "commit": "e" * 40, + page = sm.generate_explorer_page({"records": 7, "count": 7, "commit": "e" * 40, "repo": REPO_URL}) self.assertIn("", page) self.assertIn("downloads/measurements.sqlite", page) + self.assertIn(f"({REPO_URL}/tree/{'e' * 40}/measurements)", page) - def test_the_deploy_clones_the_measurements_at_the_pin(self) -> None: - workflow = (REPO_ROOT / ".github" / "workflows" / "deploy-site.yml").read_text() - clone = workflow.index("Clone the measurements at their pin") - step = workflow[clone:workflow.index("- name:", clone)] - self.assertIn("measurements-pin.json", step) - self.assertIn('fetch -q --depth 1 "$repo" "$commit"', step) - self.assertIn('checkout -q --detach "$commit"', step) - self.assertNotIn("uses:", step) - self.assertLess(clone, workflow.index("Validate data contracts")) - self.assertLess(clone, workflow.index("python scripts/generate_site.py --require-measurements")) - self.assertIn('- "measurements-pin.json"', workflow) - pin = sm.load_pin() - self.assertEqual(pin["repo"], REPO_URL) + + def test_the_builds_read_the_records_in_this_repository(self) -> None: + for name in ("deploy-site.yml", "validate.yml"): + workflow = (REPO_ROOT / ".github" / "workflows" / name).read_text() + with self.subTest(workflow=name): + self.assertIn('- "measurements/**"', workflow) + self.assertNotIn("measurements-pin", workflow) + self.assertNotIn(".cache/measurements", workflow) + self.assertNotIn("--require-measurements", workflow) + self.assertFalse((REPO_ROOT / "measurements-pin.json").exists()) + self.assertFalse((REPO_ROOT / "scripts" / "sync_measurements.py").exists()) + + def test_nothing_links_the_former_measurements_repository(self) -> None: + former = "retrobios-" + "measurements" + roots = ["wiki", "schemas", "scripts", "website/src", ".github", "tests", "measurements"] + suffixes = {".md", ".py", ".json", ".jsonl", ".yml", ".astro", ".ts", ".mjs"} + stray = [path.relative_to(REPO_ROOT).as_posix() + for root in roots for path in sorted((REPO_ROOT / root).rglob("*")) + if path.is_file() and path.suffix in suffixes + and former in path.read_text(encoding="utf-8", errors="replace")] + stray += [name for name in ("README.md", "CONTRIBUTING.md") + if former in (REPO_ROOT / name).read_text(encoding="utf-8")] + self.assertEqual(stray, []) class Freshness(unittest.TestCase): - def test_pin_against_the_remote_head(self) -> None: - import check_freshness as cf # noqa: PLC0415 + def test_no_measurements_area(self) -> None: + import check_freshness # noqa: PLC0415 + import refresh_stale # noqa: PLC0415 - pin = sm.load_pin() - self.assertEqual(cf.check_measurements(False, head=lambda _repo: pin["commit"]).status, cf.OK) - stale = cf.check_measurements(False, head=lambda _repo: "f" * 40) - self.assertEqual(stale.status, cf.STALE) - self.assertIn("sync_measurements.py --fetch --bump", stale.detail) - self.assertEqual(cf.check_measurements(True).status, cf.SKIPPED) - self.assertIn("measurements", cf.AREAS) + self.assertNotIn("measurements", check_freshness.AREAS) + self.assertNotIn("measurements", refresh_stale.AUTO_AREAS) if __name__ == "__main__": diff --git a/tests/test_site_r4_reference.py b/tests/test_site_r4_reference.py index 874445320..094880d45 100644 --- a/tests/test_site_r4_reference.py +++ b/tests/test_site_r4_reference.py @@ -248,32 +248,23 @@ class SourceLinksOnlyWhereTheFileIs(unittest.TestCase): def test_every_rendered_link_is_recorded(self): record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*")) - import sync_measurements # noqa: PLC0415 + import measurement_records # noqa: PLC0415 - # A Tested block links the measurements repository at its pin. The - # record of source trees does not cover it: those paths are checked - # against the pinned tree of the clone itself. - measured_slug = urllib.parse.urlsplit(sync_measurements.load_pin()["repo"]).path.strip("/") - measured: dict[str, set[str]] = {} + # A Tested block links a record line and its data file in this + # repository. The records are append-only, so a file a permalink + # names is still in the tree. + records = set(measurement_records.tree_files(REPO_ROOT)) checked, stray = 0, [] for page in pages: for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")): - if "/Abdess/retrobios/" in found[0]: - continue key = f"{found['host']}/{found['slug']}@{found['pin']}" path = urllib.parse.unquote(found["path"]) - checked += 1 - if found["slug"] == measured_slug: - if found["pin"] not in measured: - if not sync_measurements.has_commit(sync_measurements.CLONE, found["pin"]): - stray.append((page.name, key, "pin not in the measurements clone")) - measured[found["pin"]] = set() - continue - measured[found["pin"]] = set( - sync_measurements.tree_files(sync_measurements.CLONE, found["pin"])) - if path not in measured[found["pin"]]: + if "/Abdess/retrobios/" in found[0]: + if path.startswith(f"{measurement_records.RECORDS_DIR}/") and path not in records: stray.append((page.name, key, path)) - elif path not in record.get(key, ()): + continue + checked += 1 + if path not in record.get(key, ()): stray.append((page.name, key, path)) self.assertGreater(checked, 1000) self.assertEqual(stray[:20], []) diff --git a/website/src/components/MeasurementsExplorer.astro b/website/src/components/MeasurementsExplorer.astro index dd4d7dd45..408c1c486 100644 --- a/website/src/components/MeasurementsExplorer.astro +++ b/website/src/components/MeasurementsExplorer.astro @@ -1,7 +1,7 @@ --- // The Measurements page: every test of the measurements index, filtered in // the browser. Reads public/data/measurements/index.json, which -// scripts/sync_measurements.py writes from the records at the pinned commit. +// scripts/sitemeasurements.py writes from the records under measurements/. // Each row keeps its method and the computer it ran on beside the result, // and links its full record. The filters answer from the index alone: no // SQL engine is loaded.