refactor: keep one full record per line

Each line of measurements/<emulator>.jsonl now holds the whole run:
every repetition, core logs, per-frame hashes and raw timings. The gzip
data files, the data schema and the split and join code are gone, and
core_probe --record appends one line. record_version moves to 3.

The seven records are rewritten once: each new line equals its old line
joined with its data file, and rebuilds the first-layout file its id
hashes.

scripts/core_probe.py

@@ -10,10 +10,9 @@ Three modes share one runner:
 
 Each repetition runs in its own process, under strace when it is installed,
 so the files a core opened are read from the kernel rather than guessed.
-`--record` appends one record per invocation to measurements/ of this
-repository, carrying what is needed to reproduce it (core build, frontend
-commit, host, inputs by hash). measurement_records.py splits it into a slim
-line and a gzip sidecar.
+`--record` appends one record per invocation, as one line of
+measurements/<emulator>.jsonl in this repository, carrying what is needed to
+reproduce it (core build, frontend commit, host, inputs by hash).
 """
 
 from __future__ import annotations
@@ -771,9 +770,8 @@ def build_parser() -> argparse.ArgumentParser:
     common.add_argument("--core-commit", help="commit the core was built from")
     common.add_argument("--build-command", help="command and flags that built the core")
     common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
-                        help="append the record to measurements/<emulator>.jsonl and write "
-                             "its data file in measurements/data/<emulator>/ of the checkout "
-                             "DIR (default: this one)")
+                        help="append the record as one line of measurements/<emulator>.jsonl "
+                             "in the checkout DIR (default: this one)")
     common.add_argument("--emulator", help="profile the record is filed under (default: the "
                                            "profile whose cores: names the .so, else the "
                                            "core's library name)")
@@ -938,24 +936,19 @@ def default_subject(record: dict) -> str:
     return f"{record['kind']}-{slugify(content, 30)}-{digest}"
 
 
-def write_record(directory: Path, full: dict) -> tuple[Path, Path]:
-    """Append the record to its emulator's .jsonl and write its sidecar."""
-    from measurement_records import (  # noqa: PLC0415
-        line_text, parse_lines, records_path, seal, sidecar_path, split)
+def write_record(directory: Path, full: dict) -> Path:
+    """Append the record, with its id, to its emulator's .jsonl."""
+    from measurement_records import line_text, parse_lines, records_path, seal  # noqa: PLC0415
 
-    slim, sidecar = split(full)
-    sealed, blob = seal(slim, sidecar)
+    sealed = seal(full)
     lines = directory / records_path(sealed["emulator"])
     existing = parse_lines(lines.read_text(encoding="utf-8"), str(lines)) if lines.is_file() else []
     if sealed["supersedes"] and sealed["supersedes"] not in {r["id"] for r in existing}:
         raise ProbeError(f"--supersedes {sealed['supersedes']}: no such record in {lines}")
-    data = directory / sidecar_path(sealed["emulator"], sealed["id"])
-    data.parent.mkdir(parents=True, exist_ok=True)
-    data.write_bytes(blob)
     lines.parent.mkdir(parents=True, exist_ok=True)
     with lines.open("a", encoding="utf-8") as handle:
         handle.write(line_text(sealed) + "\n")
-    return lines, data
+    return lines
 
 
 def probe(args: argparse.Namespace) -> int:
@@ -1032,11 +1025,11 @@ def probe(args: argparse.Namespace) -> int:
     record["subject"] = args.subject or default_subject(record)
     if args.record:
         try:
-            lines, data = write_record(Path(args.record), record)
+            lines = write_record(Path(args.record), record)
         except ProbeError as exc:
             print(f"ERROR: {exc}", file=sys.stderr)
             return 2
-        print(f"recorded {lines} (line {sum(1 for _ in lines.open(encoding='utf-8'))}) and {data}",
+        print(f"recorded {lines} (line {sum(1 for _ in lines.open(encoding='utf-8'))})",
               file=sys.stderr)
     print(json.dumps(record, indent=2, sort_keys=False))
     return 0

scripts/measurement_records.py

@@ -1,22 +1,17 @@
 """Core probe records as this repository stores them.
 
-A run of core_probe.py produces one full record. It is stored as two parts:
+A run of core_probe.py produces one full record, stored as one line:
 
-  measurements/<emulator>.jsonl                one slim record per line, append-only
-  measurements/data/<emulator>/<id>.json.gz    the bulky rest, named by the record id
+  measurements/<emulator>.jsonl    one record per line, append-only
 
-The slim record keeps what a reader needs to judge and reproduce the result:
-claim, method, observable, result, statistics, host, inputs by hash, core
-build and frontend commit, and per variant the values of its first
-repetition. The sidecar holds every repetition in full (per-frame hashes,
-core logs, declared options), the raw timing values the statistics were
-computed from, and any compared value too large to sit in a line. Its
-sha256 is in the record, so the line vouches for it. ``join`` puts the two
-back into the full record, ``split`` is its inverse.
+A line holds the whole run: claim, method, observable, result, host, inputs
+by hash, core build and frontend commit, and every repetition of every
+variant in full (per-frame hashes, core logs, declared options), with the
+raw timing values the statistics were computed from.
 
 A record id is the date and the first 12 hex digits of the sha256 of the
-record without its id. A record carried over from the first layout keeps
-the id it had, which hashes that full record.
+record without its id, in canonical JSON. A record carried over from the
+first layout keeps the id it had, which hashes that full record.
 
 Standard library only: core_probe.py, validate_schemas.py and the site
 read it.
@@ -24,8 +19,6 @@ read it.
 
 from __future__ import annotations
 
-import copy
-import gzip
 import hashlib
 import json
 import re
@@ -34,19 +27,14 @@ from collections.abc import Iterable
 from dataclasses import dataclass
 from pathlib import Path
 
-RECORD_VERSION = 2
+RECORD_VERSION = 3
 RECORDS_DIR = "measurements"
-DATA_DIR = f"{RECORDS_DIR}/data"
-# Fields a repetition keeps in the slim record, per variant, from the first one.
-VARIANT_SUMMARY = ("observables", "files_opened", "files_probed_absent", "tracing")
-# Observables too large for a line: their values stay in the sidecar.
-BULKY_OBSERVABLES = ("frame_hashes",)
 ID_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2}-[0-9a-f]{12}$")
 SLUG_PATTERN = re.compile(r"^[a-z0-9]+(?:[-_][a-z0-9]+)*$")
 
 
 class RecordError(ValueError):
-    """A record or sidecar that does not match its counterpart."""
+    """A record line that cannot be read."""
 
 
 def canonical(value: object) -> bytes:
@@ -57,6 +45,11 @@ def sha256_hex(data: bytes) -> str:
     return hashlib.sha256(data).hexdigest()
 
 
+def value_sha1(value: object) -> str:
+    """The sha1 of a value in canonical JSON."""
+    return hashlib.sha1(canonical(value)).hexdigest()
+
+
 def slugify(text: str, limit: int = 40) -> str:
     """Lowercase words joined by hyphens, cut at a word boundary."""
     words = re.findall(r"[a-z0-9]+", text.lower())
@@ -69,151 +62,55 @@ def slugify(text: str, limit: int = 40) -> str:
     return out or "record"
 
 
-def record_id(slim: dict) -> str:
-    body = {key: value for key, value in slim.items() if key != "id"}
-    return f"{slim['recorded_at'][:10]}-{sha256_hex(canonical(body))[:12]}"
+def record_id(record: dict) -> str:
+    body = {key: value for key, value in record.items() if key != "id"}
+    return f"{record['recorded_at'][:10]}-{sha256_hex(canonical(body))[:12]}"
 
 
-def line_text(slim: dict) -> str:
+def seal(record: dict) -> dict:
+    """The record with its id."""
+    return {**record, "id": record_id(record)}
+
+
+def line_text(record: dict) -> str:
     """The record as one line of its .jsonl file, keys in a fixed order."""
-    return json.dumps(slim, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
-
-
-def sidecar_path(emulator: str, rid: str) -> str:
-    return f"{DATA_DIR}/{emulator}/{rid}.json.gz"
+    return json.dumps(record, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
 
 
 def records_path(emulator: str) -> str:
     return f"{RECORDS_DIR}/{emulator}.jsonl"
 
 
-def sidecar_bytes(sidecar: dict) -> bytes:
-    """gzip of the sidecar JSON, with no name and no time in the header."""
-    return gzip.compress(canonical(sidecar), compresslevel=9, mtime=0)
+IDENTITY = ("id", "emulator", "subject", "supersedes", "migrated_from_version")
 
 
-def read_sidecar(blob: bytes) -> dict:
-    return json.loads(gzip.decompress(blob))
-
-
-def _value_digest(value: object) -> str:
-    return hashlib.sha1(canonical(value)).hexdigest()
-
-
-def split(full: dict) -> tuple[dict, dict]:
-    """The full record as (slim record without id and data, sidecar)."""
-    slim = copy.deepcopy(full)
-    sidecar: dict = {"variants": [], "timing_raw": {}, "group_values": {}}
-    variants = []
-    for variant in slim["variants"]:
-        reps = variant["repetitions"]
-        sidecar["variants"].append({"label": variant["label"], "repetitions": reps})
-        first = reps[0]
-        summary = {key: first.get(key) for key in VARIANT_SUMMARY}
-        summary["observables"] = {
-            key: value for key, value in (first.get("observables") or {}).items()
-            if key not in BULKY_OBSERVABLES
-        }
-        variants.append({
-            "label": variant["label"],
-            "system_files": variant["system_files"],
-            "flags": variant["flags"],
-            "repetitions": len(reps),
-            "loaded": all(rep.get("loaded") for rep in reps),
-            **summary,
-        })
-    slim["variants"] = variants
-    for name, stats in (slim.get("timing") or {}).items():
-        sidecar["timing_raw"][name] = stats.pop("raw")
-    for group in ((slim.get("check") or {}).get("result") or {}).get("groups", []):
-        if isinstance(group.get("value"), (list, dict)):
-            digest = _value_digest(group["value"])
-            sidecar["group_values"][digest] = group.pop("value")
-            group["value_sha1"] = digest
-    return slim, sidecar
-
-
-def join(slim: dict, sidecar: dict) -> dict:
-    """The full record a slim record and its sidecar were split from."""
-    full = copy.deepcopy(slim)
-    full.pop("id", None)
-    full.pop("data", None)
-    labels = [v["label"] for v in full["variants"]]
-    if labels != [v["label"] for v in sidecar["variants"]]:
-        raise RecordError(f"variants differ: record {labels}, sidecar "
-                          f"{[v['label'] for v in sidecar['variants']]}")
-    full["variants"] = [
-        {"label": variant["label"], "system_files": variant["system_files"],
-         "repetitions": copy.deepcopy(stored["repetitions"]), "flags": variant["flags"]}
-        for variant, stored in zip(full["variants"], sidecar["variants"])
-    ]
-    for name, raw in sidecar.get("timing_raw", {}).items():
-        full["timing"][name]["raw"] = list(raw)
-    for group in ((full.get("check") or {}).get("result") or {}).get("groups", []):
-        digest = group.pop("value_sha1", None)
-        if digest is not None:
-            group["value"] = copy.deepcopy(sidecar["group_values"][digest])
-    return full
-
-
-def seal(slim: dict, sidecar: dict, rid: str | None = None) -> tuple[dict, bytes]:
-    """The slim record with its data reference and id, and the sidecar bytes.
-
-    ``rid`` keeps an id given before (a record carried over from the first
-    layout), otherwise the id is computed.
-    """
-    blob = sidecar_bytes(sidecar)
-    sealed = {**slim, "data": {"sha256": sha256_hex(blob), "bytes": len(blob)}}
-    sealed["id"] = rid or record_id(sealed)
-    return sealed, blob
-
-
-def check_pair(slim: dict, blob: bytes) -> list[str]:
-    """What is wrong between a stored line and its stored sidecar."""
-    problems = []
-    data = slim.get("data") or {}
-    if sha256_hex(blob) != data.get("sha256"):
-        problems.append(f"{slim.get('id')}: sidecar sha256 {sha256_hex(blob)} != {data.get('sha256')}")
-    if len(blob) != data.get("bytes"):
-        problems.append(f"{slim.get('id')}: sidecar is {len(blob)} bytes, record says {data.get('bytes')}")
-    if not problems:
-        try:
-            full = join(slim, read_sidecar(blob))
-        except (RecordError, KeyError, OSError, ValueError) as exc:
-            return [f"{slim.get('id')}: record and sidecar do not join: {exc}"]
-        if slim.get("migrated_from_version") == 1:
-            digest = sha256_hex(canonical(first_layout(full)))[:12]
-            if str(slim.get("id", "")).rsplit("-", 1)[-1] != digest:
-                problems.append(f"{slim.get('id')}: id does not hash the first-layout record ({digest})")
-    return problems
-
-
-V2_IDENTITY = ("emulator", "subject", "supersedes", "migrated_from_version")
-
-
-def first_layout(full: dict) -> dict:
-    """A full record as the first layout wrote it: no emulator, subject or
+def first_layout(record: dict) -> dict:
+    """A record as the first layout wrote it: no id, emulator, subject or
     supersedes, record_version 1."""
-    old = {key: value for key, value in full.items() if key not in V2_IDENTITY}
+    old = {key: value for key, value in record.items() if key not in IDENTITY}
     old["record_version"] = 1
     return old
 
 
-def check_line(slim: dict) -> list[str]:
+def check_line(record: dict) -> list[str]:
     """The structural rules a line holds, without a JSON Schema validator."""
-    rid = slim.get("id", "")
+    rid = record.get("id", "")
     problems = []
-    if slim.get("record_version") != RECORD_VERSION:
-        problems.append(f"{rid}: record_version {slim.get('record_version')} != {RECORD_VERSION}")
+    if record.get("record_version") != RECORD_VERSION:
+        problems.append(f"{rid}: record_version {record.get('record_version')} != {RECORD_VERSION}")
     if not ID_PATTERN.match(str(rid)):
         problems.append(f"{rid!r}: not an id of the form YYYY-MM-DD-<12 hex>")
-    elif str(slim.get("recorded_at", ""))[:10] != rid[:10]:
-        problems.append(f"{rid}: id date differs from recorded_at {slim.get('recorded_at')}")
-    elif slim.get("migrated_from_version") is None and record_id(slim) != rid:
-        problems.append(f"{rid}: id does not hash the record ({record_id(slim)})")
+    elif str(record.get("recorded_at", ""))[:10] != rid[:10]:
+        problems.append(f"{rid}: id date differs from recorded_at {record.get('recorded_at')}")
+    elif record.get("migrated_from_version") == 1:
+        digest = sha256_hex(canonical(first_layout(record)))[:12]
+        if not rid.endswith(f"-{digest}"):
+            problems.append(f"{rid}: id does not hash the first-layout record ({digest})")
+    elif record_id(record) != rid:
+        problems.append(f"{rid}: id does not hash the record ({record_id(record)})")
     for key in ("emulator", "subject"):
-        if not SLUG_PATTERN.match(str(slim.get(key, ""))):
-            problems.append(f"{rid}: {key} {slim.get(key)!r} is not a lowercase slug")
+        if not SLUG_PATTERN.match(str(record.get(key, ""))):
+            problems.append(f"{rid}: {key} {record.get(key)!r} is not a lowercase slug")
     return problems
 
 
@@ -262,7 +159,7 @@ class Stored:
 
 
 def tree_files(root: Path) -> dict[str, Path]:
-    """Repository path -> file of every record file and data file under ``root``."""
+    """Repository path -> file of everything under ``root``/measurements."""
     base = root / RECORDS_DIR
     if not base.is_dir():
         return {}
@@ -271,16 +168,14 @@ def tree_files(root: Path) -> dict[str, Path]:
 
 
 def read_records(root: Path) -> list[Stored]:
-    """Every record line under ``root``, each checked against its rules and data."""
+    """Every record line under ``root``, each checked against its rules."""
     files = tree_files(root)
-    data_prefix = f"{DATA_DIR}/"
     problems: list[str] = []
     stored: list[Stored] = []
-    claimed: set[str] = set()
-    for path in sorted(p for p in files if not p.startswith(data_prefix)):
+    for path in sorted(files):
         emulator = path[len(RECORDS_DIR) + 1:].removesuffix(".jsonl")
         if not path.endswith(".jsonl") or "/" in emulator:
-            problems.append(f"{path}: {RECORDS_DIR}/ holds <emulator>.jsonl files and data/ only")
+            problems.append(f"{path}: {RECORDS_DIR}/ holds <emulator>.jsonl files only")
             continue
         seen: set[str] = set()
         try:
@@ -298,15 +193,7 @@ def read_records(root: Path) -> list[Stored]:
             if record.get("supersedes") and record["supersedes"] not in seen:
                 problems.append(f"{where}: supersedes {record['supersedes']}, no earlier line of this file")
             seen.add(record.get("id"))
-            data = sidecar_path(emulator, str(record.get("id")))
-            claimed.add(data)
-            if data not in files:
-                problems.append(f"{where}: {data} is missing")
-            else:
-                problems.extend(f"{where}: {p}" for p in check_pair(record, files[data].read_bytes()))
             stored.append(Stored(emulator, number, record))
-    problems.extend(f"{path}: no record line names it"
-                    for path in sorted(files) if path.startswith(data_prefix) and path not in claimed)
     if problems:
         raise MeasurementsError("records under {}:\n  {}".format(root, "\n  ".join(problems)))
     return stored
@@ -317,7 +204,7 @@ def _git(root: Path, *args: str) -> subprocess.CompletedProcess:
 
 
 def committed_files(root: Path, ref: str) -> dict[str, bytes]:
-    """Path -> content of every record and data file at ``ref``."""
+    """Path -> content of every file under measurements/ at ``ref``."""
     listing = _git(root, "ls-tree", "-r", "-z", "--name-only", ref, "--", RECORDS_DIR)
     if listing.returncode != 0:
         raise MeasurementsError(f"git ls-tree {ref}: {listing.stderr.decode(errors='replace').strip()}")
@@ -331,8 +218,8 @@ def committed_files(root: Path, ref: str) -> dict[str, bytes]:
 
 
 def append_only_problems(root: Path, ref: str = "HEAD") -> list[str]:
-    """What the tree under ``root`` changed or removed of the lines and data
-    files ``ref`` holds. Lines added at the end and new files are allowed."""
+    """What the tree under ``root`` changed or removed of the lines ``ref``
+    holds. Lines added at the end and new files are allowed."""
     before, after = committed_files(root, ref), tree_files(root)
     problems = []
     for path, content in sorted(before.items()):

scripts/sitemeasurements.py

@@ -1,8 +1,8 @@
 """The measurements the site shows, from the records under measurements/.
 
-The records are core probe runs, one append-only .jsonl per emulator with a
-gzip data file per record (measurement_records.py). From them the build
-writes, never committed:
+The records are core probe runs, one append-only .jsonl per emulator with one
+full record per line (measurement_records.py). From them the build writes,
+never committed:
 
   public/data/measurements/index.json        the compact index the explorer and
                                              the emulator pages read, one item
@@ -39,7 +39,9 @@ HOSTS_MARKER = "<!-- measurements-hosts -->"
 INDEX_PATH = Path("data") / "measurements" / "index.json"
 SQLITE_PATH = Path("downloads") / "measurements.sqlite"
 PER_EMULATOR_DIR = Path("downloads") / "measurements"
-SQLITE_SCHEMA_VERSION = "1"
+SQLITE_SCHEMA_VERSION = "2"
+# Observables the variants table leaves out: one value per sampled frame.
+BULKY_OBSERVABLES = ("frame_hashes",)
 INDEX_SCHEMA_VERSION = 1
 
 
@@ -109,7 +111,6 @@ def index_item(entry: mr.Stored, runs: int, source: dict, profiles: dict,
         "id": record["id"],
         "runs": runs,
         "record_url": permalink(source["repo"], source["commit"], mr.records_path(entry.emulator), entry.line),
-        "data_url": permalink(source["repo"], source["commit"], mr.sidecar_path(entry.emulator, record["id"])),
         "sqlite": f"downloads/measurements/{entry.emulator}.sqlite",
     }
 
@@ -164,8 +165,8 @@ SCHEMA_SQL = """
         superseded INTEGER NOT NULL, core_sha256 TEXT NOT NULL REFERENCES cores(sha256),
         host_id INTEGER NOT NULL REFERENCES hosts(id), load_before TEXT, load_after TEXT,
         max_load REAL, frontend_commit TEXT, script_sha256 TEXT NOT NULL,
-        script_modified INTEGER, flags TEXT NOT NULL, data_sha256 TEXT NOT NULL,
-        data_bytes INTEGER NOT NULL, record_url TEXT NOT NULL, line INTEGER NOT NULL,
+        script_modified INTEGER, flags TEXT NOT NULL, record_url TEXT NOT NULL,
+        line INTEGER NOT NULL,
         migrated_from_version INTEGER
     );
     CREATE TABLE inputs (
@@ -239,7 +240,7 @@ def write_sqlite(destination: Path, stored: list[mr.Stored], source: dict | None
             "measurements_repo": (source or {}).get("repo") or "",
             "measurements_commit": (source or {}).get("commit") or "",
             "license": "MIT",
-            "scope": "core probe records: the record lines, not their data files",
+            "scope": "core probe records, one per line of measurements/<emulator>.jsonl",
         }
         connection.executemany("INSERT INTO meta VALUES (?, ?)", sorted(meta.items()))
         cores = {s.record["core"]["sha256"]: s.record["core"] for s in stored}
@@ -268,13 +269,13 @@ def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
     r = entry.record
     host, front, inputs = r["host"], r["frontend"], r["inputs"]
     rid = r["id"]
-    connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
+    connection.execute("INSERT INTO runs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
         rid, entry.emulator, r["subject"], r["kind"], r["recorded_at"], r.get("supersedes"),
         int(superseded), r["core"]["sha256"], hosts[_host_key(host)],
         json_text(host.get("load_before")), json_text(host.get("load_after")), host.get("max_load"),
         front.get("commit"), front["script_sha256"],
         None if front.get("script_modified") is None else int(front["script_modified"]),
-        json_text(r["flags"]), r["data"]["sha256"], r["data"]["bytes"],
+        json_text(r["flags"]),
         permalink(source.get("repo", ""), source.get("commit", ""), mr.records_path(entry.emulator), entry.line),
         entry.line, r.get("migrated_from_version")))
     content = inputs.get("content") or {}
@@ -293,15 +294,21 @@ def _insert_run(connection: sqlite3.Connection, entry: mr.Stored, source: dict,
             rid, check["claim"], check["method"], check["observable"], check["variable"],
             result["status"], result["statement"]))
         connection.executemany("INSERT INTO check_groups VALUES (?, ?, ?, ?, ?)", [
-            (rid, position, json_text(group["value"]) if "value" in group else None,
-             group.get("value_sha1"), json_text(group["variants"]))
+            (rid, position, json_text(group["value"]),
+             mr.value_sha1(group["value"]) if isinstance(group["value"], (list, dict)) else None,
+             json_text(group["variants"]))
             for position, group in enumerate(result["groups"])])
     for position, variant in enumerate(r["variants"]):
+        reps = variant["repetitions"]
+        first = reps[0]
+        # The first repetition stands for the variant, its per-frame hashes left out.
+        observables = {key: value for key, value in (first.get("observables") or {}).items()
+                       if key not in BULKY_OBSERVABLES}
         connection.execute("INSERT INTO variants VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", (
-            rid, position, variant["label"], variant["repetitions"], int(variant["loaded"]),
-            json_text(variant["flags"]), json_text(variant["observables"]),
-            json_text(variant["files_opened"]), json_text(variant["files_probed_absent"]),
-            json_text(variant["tracing"])))
+            rid, position, variant["label"], len(reps), int(all(rep.get("loaded") for rep in reps)),
+            json_text(variant["flags"]), json_text(observables),
+            json_text(first.get("files_opened")), json_text(first.get("files_probed_absent")),
+            json_text(first.get("tracing"))))
         connection.executemany("INSERT INTO system_files VALUES (?, ?, ?, ?)", [
             (rid, position, dest, sha1) for dest, sha1 in sorted(variant["system_files"].items())])
     for metric, stats in sorted((r.get("timing") or {}).items()):
@@ -411,7 +418,7 @@ def tested_item(item: dict) -> list[str]:
         + (f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" if item["flags"] else "")
         + ".",
         "",
-        f"    [Full record]({item['record_url']}), [its data]({item['data_url']}), {runs}.",
+        f"    [Full record]({item['record_url']}), {runs}.",
         "",
     ]
     return lines
@@ -503,8 +510,7 @@ def data_page_section(path: Path) -> list[str]:
         "`downloads/measurements/<emulator>.sqlite` the records of one emulator, linked "
         f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads "
         "[`data/measurements/index.json`](data/measurements/index.json). A record line is "
-        "valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json) "
-        "and its data file against [`core-probe-data.schema.json`](api/v1/schemas/core-probe-data.schema.json).",
+        "valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json).",
         "",
         "| Table | Columns |",
         "|-------|---------|",

scripts/validate_schemas.py

@@ -594,27 +594,23 @@ def _format_errors() -> list[str]:
 
 
 def _measurement_errors(root: Path | None = None) -> list[str]:
-    """Every record line and data file under measurements/, against their
-    schemas and the rules measurement_records.read_records applies."""
+    """Every record line under measurements/, against its schema and the
+    rules measurement_records.read_records applies."""
     import measurement_records as mr  # noqa: PLC0415
 
     root = root or ROOT
-    record_validator = _validator("core-probe-record.schema.json")
-    data_validator = _validator("core-probe-data.schema.json")
+    validator = _validator("core-probe-record.schema.json")
     out: list[str] = []
-    files = mr.tree_files(root)
-    for path in sorted(p for p in files if p.endswith(".jsonl")):
-        emulator = path[len(mr.RECORDS_DIR) + 1:].removesuffix(".jsonl")
-        for number, line in enumerate(files[path].read_text(encoding="utf-8").splitlines(), 1):
+    for path, file in sorted(mr.tree_files(root).items()):
+        if not path.endswith(".jsonl"):
+            continue
+        for number, line in enumerate(file.read_text(encoding="utf-8").splitlines(), 1):
             try:
                 record = json.loads(line)
             except json.JSONDecodeError as exc:
                 out.append(f"{path}:{number}: {exc}")
                 continue
-            out.extend(_errors(record_validator, record, f"{path}:{number}"))
-            data = mr.sidecar_path(emulator, str(record.get("id")))
-            if data in files:
-                out.extend(_errors(data_validator, mr.read_sidecar(files[data].read_bytes()), data))
+            out.extend(_errors(validator, record, f"{path}:{number}"))
     if not out:
         try:
             mr.read_records(root)