refactor: keep one full record per line

Each line of measurements/<emulator>.jsonl now holds the whole run:
every repetition, core logs, per-frame hashes and raw timings. The gzip
data files, the data schema and the split and join code are gone, and
core_probe --record appends one line. record_version moves to 3.

The seven records are rewritten once: each new line equals its old line
joined with its data file, and rebuilds the first-layout file its id
hashes.
This commit is contained in:
Abdessamad Derraz committed 2026-10-27 09:57:39 +01:00
1 parent 9c34e8891f
commit da68f19411
21 files changed
+334 -659

No files matched your search

+8 -12
View File
@@ -594,27 +594,23 @@ def _format_errors() -> list[str]:
def _measurement_errors(root: Path | None = None) -> list[str]:
"""Every record line and data file under measurements/, against their
schemas and the rules measurement_records.read_records applies."""
"""Every record line under measurements/, against its schema and the
rules measurement_records.read_records applies."""
import measurement_records as mr # noqa: PLC0415
root = root or ROOT
record_validator = _validator("core-probe-record.schema.json")
data_validator = _validator("core-probe-data.schema.json")
validator = _validator("core-probe-record.schema.json")
out: list[str] = []
files = mr.tree_files(root)
for path in sorted(p for p in files if p.endswith(".jsonl")):
emulator = path[len(mr.RECORDS_DIR) + 1:].removesuffix(".jsonl")
for number, line in enumerate(files[path].read_text(encoding="utf-8").splitlines(), 1):
for path, file in sorted(mr.tree_files(root).items()):
if not path.endswith(".jsonl"):
continue
for number, line in enumerate(file.read_text(encoding="utf-8").splitlines(), 1):
try:
record = json.loads(line)
except json.JSONDecodeError as exc:
out.append(f"{path}:{number}: {exc}")
continue
out.extend(_errors(record_validator, record, f"{path}:{number}"))
data = mr.sidecar_path(emulator, str(record.get("id")))
if data in files:
out.extend(_errors(data_validator, mr.read_sidecar(files[data].read_bytes()), data))
out.extend(_errors(validator, record, f"{path}:{number}"))
if not out:
try:
mr.read_records(root)