mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 07:33:45 -06:00
refactor: keep one full record per line
Each line of measurements/<emulator>.jsonl now holds the whole run: every repetition, core logs, per-frame hashes and raw timings. The gzip data files, the data schema and the split and join code are gone, and core_probe --record appends one line. record_version moves to 3. The seven records are rewritten once: each new line equals its old line joined with its data file, and rebuilds the first-layout file its id hashes.
This commit is contained in:
1 parent
9c34e8891f
commit
da68f19411
21 files changed
+334
-659
No files matched your search
+12
-19
@@ -10,10 +10,9 @@ Three modes share one runner:
|
||||
|
||||
Each repetition runs in its own process, under strace when it is installed,
|
||||
so the files a core opened are read from the kernel rather than guessed.
|
||||
`--record` appends one record per invocation to measurements/ of this
|
||||
repository, carrying what is needed to reproduce it (core build, frontend
|
||||
commit, host, inputs by hash). measurement_records.py splits it into a slim
|
||||
line and a gzip sidecar.
|
||||
`--record` appends one record per invocation, as one line of
|
||||
measurements/<emulator>.jsonl in this repository, carrying what is needed to
|
||||
reproduce it (core build, frontend commit, host, inputs by hash).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -771,9 +770,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
common.add_argument("--core-commit", help="commit the core was built from")
|
||||
common.add_argument("--build-command", help="command and flags that built the core")
|
||||
common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
|
||||
help="append the record to measurements/<emulator>.jsonl and write "
|
||||
"its data file in measurements/data/<emulator>/ of the checkout "
|
||||
"DIR (default: this one)")
|
||||
help="append the record as one line of measurements/<emulator>.jsonl "
|
||||
"in the checkout DIR (default: this one)")
|
||||
common.add_argument("--emulator", help="profile the record is filed under (default: the "
|
||||
"profile whose cores: names the .so, else the "
|
||||
"core's library name)")
|
||||
@@ -938,24 +936,19 @@ def default_subject(record: dict) -> str:
|
||||
return f"{record['kind']}-{slugify(content, 30)}-{digest}"
|
||||
|
||||
|
||||
def write_record(directory: Path, full: dict) -> tuple[Path, Path]:
|
||||
"""Append the record to its emulator's .jsonl and write its sidecar."""
|
||||
from measurement_records import ( # noqa: PLC0415
|
||||
line_text, parse_lines, records_path, seal, sidecar_path, split)
|
||||
def write_record(directory: Path, full: dict) -> Path:
|
||||
"""Append the record, with its id, to its emulator's .jsonl."""
|
||||
from measurement_records import line_text, parse_lines, records_path, seal # noqa: PLC0415
|
||||
|
||||
slim, sidecar = split(full)
|
||||
sealed, blob = seal(slim, sidecar)
|
||||
sealed = seal(full)
|
||||
lines = directory / records_path(sealed["emulator"])
|
||||
existing = parse_lines(lines.read_text(encoding="utf-8"), str(lines)) if lines.is_file() else []
|
||||
if sealed["supersedes"] and sealed["supersedes"] not in {r["id"] for r in existing}:
|
||||
raise ProbeError(f"--supersedes {sealed['supersedes']}: no such record in {lines}")
|
||||
data = directory / sidecar_path(sealed["emulator"], sealed["id"])
|
||||
data.parent.mkdir(parents=True, exist_ok=True)
|
||||
data.write_bytes(blob)
|
||||
lines.parent.mkdir(parents=True, exist_ok=True)
|
||||
with lines.open("a", encoding="utf-8") as handle:
|
||||
handle.write(line_text(sealed) + "\n")
|
||||
return lines, data
|
||||
return lines
|
||||
|
||||
|
||||
def probe(args: argparse.Namespace) -> int:
|
||||
@@ -1032,11 +1025,11 @@ def probe(args: argparse.Namespace) -> int:
|
||||
record["subject"] = args.subject or default_subject(record)
|
||||
if args.record:
|
||||
try:
|
||||
lines, data = write_record(Path(args.record), record)
|
||||
lines = write_record(Path(args.record), record)
|
||||
except ProbeError as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
print(f"recorded {lines} (line {sum(1 for _ in lines.open(encoding='utf-8'))}) and {data}",
|
||||
print(f"recorded {lines} (line {sum(1 for _ in lines.open(encoding='utf-8'))})",
|
||||
file=sys.stderr)
|
||||
print(json.dumps(record, indent=2, sort_keys=False))
|
||||
return 0
|
||||
|
||||
Reference in new issue
Block a user