mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 15:43:45 -06:00
feat: add headless libretro core probe
core_probe.py loads a libretro core with no window, runs it for a number of frames and reports the files it opened (strace), its log lines, frame, audio and save-state hashes and its speed. bench repeats a run after warm-up and gives median, mean, deviation and range; check swaps one system file between runs and compares one observable. --record writes a JSON record with the core build, frontend commit, host and every input by hash, validated against core-probe-record.schema.json. A busy host is refused, and a repetition that disagrees with the first is flagged.
This commit is contained in:
Abdessamad Derraz committed
1 parent
625bee35cc
commit
00b034612f
6 files changed
+1514
No files matched your search
@@ -0,0 +1,254 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-record.schema.json",
|
||||
"title": "RetroBIOS core probe record",
|
||||
"description": "One run of scripts/core_probe.py: a libretro core executed headless, with what is needed to reproduce it and what was measured.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["record_version", "kind", "recorded_at", "core", "frontend", "host", "inputs", "variants", "flags"],
|
||||
"properties": {
|
||||
"record_version": { "const": 1 },
|
||||
"kind": { "enum": ["run", "bench", "check"] },
|
||||
"recorded_at": { "type": "string", "format": "date-time" },
|
||||
"core": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["file", "sha256", "repo", "commit", "build_command", "compiler", "library_name", "library_version"],
|
||||
"properties": {
|
||||
"file": { "type": "string", "minLength": 1 },
|
||||
"sha256": { "$ref": "#/$defs/sha256" },
|
||||
"repo": { "type": ["string", "null"], "format": "uri" },
|
||||
"commit": { "type": ["string", "null"], "pattern": "^[0-9a-f]{7,40}$" },
|
||||
"build_command": { "type": ["string", "null"] },
|
||||
"compiler": { "type": ["array", "null"], "items": { "type": "string" } },
|
||||
"library_name": { "type": ["string", "null"] },
|
||||
"library_version": { "type": ["string", "null"] }
|
||||
}
|
||||
},
|
||||
"frontend": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["script", "commit", "script_modified", "script_sha256"],
|
||||
"properties": {
|
||||
"script": { "const": "scripts/core_probe.py" },
|
||||
"commit": { "type": ["string", "null"], "pattern": "^[0-9a-f]{40}$" },
|
||||
"script_modified": { "type": ["boolean", "null"] },
|
||||
"script_sha256": { "$ref": "#/$defs/sha256" }
|
||||
}
|
||||
},
|
||||
"host": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["cpu_model", "cpu_count", "os", "kernel", "machine", "python", "load_before", "load_after", "max_load"],
|
||||
"properties": {
|
||||
"cpu_model": { "type": ["string", "null"] },
|
||||
"cpu_count": { "type": ["integer", "null"], "minimum": 1 },
|
||||
"os": { "type": "string" },
|
||||
"kernel": { "type": "string" },
|
||||
"machine": { "type": "string" },
|
||||
"python": { "type": "string" },
|
||||
"load_before": { "$ref": "#/$defs/load" },
|
||||
"load_after": { "$ref": "#/$defs/load" },
|
||||
"max_load": { "type": "number", "minimum": 0 }
|
||||
}
|
||||
},
|
||||
"inputs": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["content", "options", "env", "frames", "warmup", "repetitions", "presses", "observables", "frame_hash_every", "joypad_ports"],
|
||||
"properties": {
|
||||
"content": {
|
||||
"oneOf": [
|
||||
{ "type": "null" },
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["label", "sha1", "size", "tracks"],
|
||||
"properties": {
|
||||
"label": { "type": ["string", "null"] },
|
||||
"sha1": { "$ref": "#/$defs/sha1" },
|
||||
"size": { "type": "integer", "minimum": 0 },
|
||||
"tracks": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["file", "sha1", "size"],
|
||||
"properties": {
|
||||
"file": { "type": "string" },
|
||||
"sha1": { "$ref": "#/$defs/sha1" },
|
||||
"size": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": { "type": "object", "additionalProperties": { "type": "string" } },
|
||||
"env": { "type": "object", "additionalProperties": { "type": "string" } },
|
||||
"joypad_ports": { "type": "integer", "minimum": 0 },
|
||||
"frames": { "type": "integer", "minimum": 1 },
|
||||
"warmup": { "type": "integer", "minimum": 0 },
|
||||
"repetitions": { "type": "integer", "minimum": 1 },
|
||||
"presses": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["frame", "button", "length"],
|
||||
"properties": {
|
||||
"frame": { "type": "integer", "minimum": 0 },
|
||||
"button": { "type": "integer", "minimum": 0, "maximum": 15 },
|
||||
"length": { "type": "integer", "minimum": 1 }
|
||||
}
|
||||
}
|
||||
},
|
||||
"observables": { "type": "array", "minItems": 1, "items": { "type": "string" } },
|
||||
"frame_hash_every": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
},
|
||||
"variants": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["label", "system_files", "repetitions", "flags"],
|
||||
"properties": {
|
||||
"label": { "type": "string", "minLength": 1 },
|
||||
"system_files": {
|
||||
"type": "object",
|
||||
"additionalProperties": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/sha1" }] }
|
||||
},
|
||||
"repetitions": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/repetition" } },
|
||||
"flags": { "type": "array", "items": { "type": "string" } }
|
||||
}
|
||||
}
|
||||
},
|
||||
"flags": { "type": "array", "items": { "type": "string" } },
|
||||
"timing": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["ms_per_frame", "fps"],
|
||||
"properties": {
|
||||
"ms_per_frame": { "$ref": "#/$defs/stats" },
|
||||
"fps": { "$ref": "#/$defs/stats" }
|
||||
}
|
||||
},
|
||||
"check": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["claim", "method", "observable", "variable", "result"],
|
||||
"properties": {
|
||||
"claim": { "type": "string", "minLength": 1 },
|
||||
"method": { "type": "string", "minLength": 1 },
|
||||
"observable": { "type": "string", "minLength": 1 },
|
||||
"variable": { "type": "string", "minLength": 1 },
|
||||
"result": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["status", "statement", "groups"],
|
||||
"properties": {
|
||||
"status": { "enum": ["same", "differs", "inconclusive"] },
|
||||
"statement": { "type": "string", "minLength": 1 },
|
||||
"groups": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["value", "variants"],
|
||||
"properties": {
|
||||
"value": {},
|
||||
"variants": { "type": "array", "minItems": 1, "items": { "type": "string" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"allOf": [
|
||||
{ "if": { "properties": { "kind": { "const": "bench" } } }, "then": { "required": ["timing"], "properties": { "inputs": { "properties": { "repetitions": { "minimum": 5 } } } } } },
|
||||
{ "if": { "properties": { "kind": { "const": "check" } } }, "then": { "required": ["check"] } }
|
||||
],
|
||||
"$defs": {
|
||||
"sha1": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
|
||||
"sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
|
||||
"load": {
|
||||
"oneOf": [
|
||||
{ "type": "null" },
|
||||
{ "type": "array", "minItems": 3, "maxItems": 3, "items": { "type": "number", "minimum": 0 } }
|
||||
]
|
||||
},
|
||||
"stats": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["median", "mean", "stdev", "min", "max", "raw"],
|
||||
"properties": {
|
||||
"median": { "type": "number" },
|
||||
"mean": { "type": "number" },
|
||||
"stdev": { "type": "number", "minimum": 0 },
|
||||
"min": { "type": "number" },
|
||||
"max": { "type": "number" },
|
||||
"raw": { "type": "array", "minItems": 1, "items": { "type": "number" } }
|
||||
}
|
||||
},
|
||||
"repetition": {
|
||||
"type": "object",
|
||||
"required": ["loaded", "library_name", "logs"],
|
||||
"properties": {
|
||||
"loaded": { "type": "boolean" },
|
||||
"library_name": { "type": "string" },
|
||||
"library_version": { "type": "string" },
|
||||
"fps_nominal": { "type": "number" },
|
||||
"frames": { "type": "integer", "minimum": 1 },
|
||||
"warmup": { "type": "integer", "minimum": 0 },
|
||||
"wall_s": { "type": "number", "minimum": 0 },
|
||||
"ms_per_frame": { "type": "number", "minimum": 0 },
|
||||
"fps": { "type": "number", "minimum": 0 },
|
||||
"observables": { "type": "object" },
|
||||
"logs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["frame", "level", "text"],
|
||||
"properties": {
|
||||
"frame": { "type": "integer", "minimum": 0 },
|
||||
"level": { "type": "string" },
|
||||
"text": { "type": "string" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"options_declared": { "type": "object" },
|
||||
"env_unanswered": { "type": "object" },
|
||||
"last_frame_size": { "type": "array", "items": { "type": "integer" } },
|
||||
"tracing": { "type": "string" },
|
||||
"input_queries": { "type": "integer", "minimum": 0 },
|
||||
"input_pressed": { "type": "integer", "minimum": 0 },
|
||||
"files_opened": {
|
||||
"oneOf": [
|
||||
{ "type": "null" },
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["path", "sha1_after", "sha1_staged"],
|
||||
"properties": {
|
||||
"path": { "type": "string" },
|
||||
"sha1_after": { "$ref": "#/$defs/sha1" },
|
||||
"sha1_staged": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/sha1" }] }
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"files_probed_absent": { "oneOf": [{ "type": "null" }, { "type": "array", "items": { "type": "string" } }] }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,930 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a libretro core headless and record what it did.
|
||||
|
||||
Three modes share one runner:
|
||||
|
||||
run one execution, printed as JSON
|
||||
bench warm-up then timed repetitions, with timing statistics
|
||||
check the same execution under several variants of one system file,
|
||||
compared on a named observable
|
||||
|
||||
Each repetition runs in its own process, under strace when it is installed,
|
||||
so the files a core opened are read from the kernel rather than guessed.
|
||||
`--record DIR` writes one JSON record per invocation, carrying what is needed
|
||||
to reproduce it (core build, frontend commit, host, inputs by hash).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import zlib
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
RECORD_VERSION = 1
|
||||
MIN_BENCH_REPETITIONS = 5
|
||||
DEFAULT_WORK_DIR = ROOT / "tmp" / "core_probe"
|
||||
BASE_OBSERVABLES = ("frame_sha1", "video_sha1", "audio_sha1", "state_sha1")
|
||||
|
||||
# libretro.h environment commands the probe answers.
|
||||
ENV_GET_CAN_DUPE = 3
|
||||
ENV_GET_SYSTEM_DIRECTORY = 9
|
||||
ENV_SET_PIXEL_FORMAT = 10
|
||||
ENV_GET_VARIABLE = 15
|
||||
ENV_SET_VARIABLES = 16
|
||||
ENV_GET_VARIABLE_UPDATE = 17
|
||||
ENV_SET_SUPPORT_NO_GAME = 18
|
||||
ENV_GET_LOG_INTERFACE = 27
|
||||
ENV_GET_SAVE_DIRECTORY = 31
|
||||
ENV_GET_CORE_OPTIONS_VERSION = 52
|
||||
ENV_EXPERIMENTAL = 0x10000
|
||||
|
||||
PIXEL_BYTES = {0: 2, 1: 4, 2: 2}
|
||||
LOG_LEVELS = ("debug", "info", "warn", "error")
|
||||
JOYPAD_BUTTONS = {
|
||||
"b": 0, "y": 1, "select": 2, "start": 3, "up": 4, "down": 5, "left": 6,
|
||||
"right": 7, "a": 8, "x": 9, "l": 10, "r": 11, "l2": 12, "r2": 13,
|
||||
"l3": 14, "r3": 15,
|
||||
}
|
||||
STRACE_LINE = re.compile(
|
||||
r'^(?:\[pid\s+)?(?P<pid>\d+)\]?\s+(?P<call>openat|open)\('
|
||||
r'(?:[^,]+,\s*)?"(?P<path>(?:[^"\\]|\\.)*)"(?:\.\.\.)?,\s*(?P<flags>[^,)]+)'
|
||||
r'.*\)\s*=\s*(?P<ret>-?\d+)(?:\s+(?P<errno>[A-Z]+))?'
|
||||
)
|
||||
PRINTF_SPEC = re.compile(r"%(?P<flags>[-+ #0]*)(?P<width>\*|\d+)?(?:\.(?P<prec>\*|\d+))?"
|
||||
r"(?P<len>hh|h|ll|l|j|z|t|L)?(?P<conv>[diouxXeEfFgGaAcspn%])")
|
||||
|
||||
|
||||
class ProbeError(RuntimeError):
|
||||
"""A run that cannot produce a trustworthy result."""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Hashing and small helpers
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def file_digest(path: Path, algorithm: str = "sha1") -> str:
|
||||
digest = hashlib.new(algorithm)
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def cue_tracks(cue: Path) -> list[Path]:
|
||||
"""Files a .cue sheet references, in order."""
|
||||
tracks: list[Path] = []
|
||||
for line in cue.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
match = re.match(r'\s*FILE\s+"([^"]+)"', line) or re.match(r"\s*FILE\s+(\S+)", line)
|
||||
if match:
|
||||
tracks.append(cue.parent / match.group(1))
|
||||
return tracks
|
||||
|
||||
|
||||
def content_identity(path: Path, label: str | None) -> dict:
|
||||
tracks = [
|
||||
{"file": track.name, "sha1": file_digest(track), "size": track.stat().st_size}
|
||||
for track in (cue_tracks(path) if path.suffix.lower() == ".cue" else [])
|
||||
]
|
||||
return {
|
||||
"label": label,
|
||||
"sha1": file_digest(path),
|
||||
"size": path.stat().st_size,
|
||||
"tracks": tracks,
|
||||
}
|
||||
|
||||
|
||||
def parse_assignment(text: str, flag: str) -> tuple[str, str]:
|
||||
key, sep, value = text.partition("=")
|
||||
if not sep or not key:
|
||||
raise argparse.ArgumentTypeError(f"{flag} expects KEY=VALUE, got {text!r}")
|
||||
return key, value
|
||||
|
||||
|
||||
def parse_press(text: str) -> tuple[int, int, int]:
|
||||
"""FRAME:BUTTON[:LENGTH] on joypad port 0."""
|
||||
parts = text.split(":")
|
||||
if len(parts) not in (2, 3) or parts[1].lower() not in JOYPAD_BUTTONS:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"--press expects FRAME:BUTTON[:LENGTH] with BUTTON in "
|
||||
f"{', '.join(JOYPAD_BUTTONS)}, got {text!r}"
|
||||
)
|
||||
try:
|
||||
frame = int(parts[0])
|
||||
length = int(parts[2]) if len(parts) == 3 else 1
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(f"--press {text!r}: {exc}") from exc
|
||||
if frame < 0 or length < 1:
|
||||
raise argparse.ArgumentTypeError(f"--press {text!r}: frame >= 0, length >= 1")
|
||||
return frame, JOYPAD_BUTTONS[parts[1].lower()], length
|
||||
|
||||
|
||||
def parse_strace(text: str) -> list[dict]:
|
||||
"""open/openat calls from `strace -f -e trace=openat,open` output."""
|
||||
calls: list[dict] = []
|
||||
for line in text.splitlines():
|
||||
match = STRACE_LINE.match(line.strip())
|
||||
if not match:
|
||||
continue
|
||||
ret = int(match.group("ret"))
|
||||
calls.append({
|
||||
"path": bytes(match.group("path"), "utf-8").decode("unicode_escape"),
|
||||
"flags": match.group("flags").strip(),
|
||||
"ok": ret >= 0,
|
||||
"errno": match.group("errno"),
|
||||
})
|
||||
return calls
|
||||
|
||||
|
||||
def files_under(calls: Iterable[dict], directory: Path) -> tuple[list[str], list[str]]:
|
||||
"""Relative paths opened, and probed but absent, under a directory."""
|
||||
base = os.path.realpath(directory)
|
||||
opened: dict[str, None] = {}
|
||||
missing: dict[str, None] = {}
|
||||
for call in calls:
|
||||
path = os.path.realpath(call["path"])
|
||||
if path != base and not path.startswith(base + os.sep):
|
||||
continue
|
||||
if os.path.isdir(path):
|
||||
continue
|
||||
rel = os.path.relpath(path, base)
|
||||
(opened if call["ok"] else missing)[rel] = None
|
||||
return list(opened), [rel for rel in missing if rel not in opened]
|
||||
|
||||
|
||||
def elf_comment(path: Path) -> list[str]:
|
||||
"""Strings of the ELF .comment section: the compilers that built the file."""
|
||||
data = path.read_bytes()
|
||||
if data[:4] != b"\x7fELF" or data[4] != 2:
|
||||
return []
|
||||
endian = "<" if data[5] == 1 else ">"
|
||||
shoff = struct.unpack_from(endian + "Q", data, 0x28)[0]
|
||||
shentsize, shnum, shstrndx = struct.unpack_from(endian + "HHH", data, 0x3A)
|
||||
sections = [struct.unpack_from(endian + "IIQQQQIIQQ", data, shoff + i * shentsize)
|
||||
for i in range(shnum)]
|
||||
if shstrndx >= len(sections):
|
||||
return []
|
||||
names_off = sections[shstrndx][4]
|
||||
for section in sections:
|
||||
name_end = data.index(b"\0", names_off + section[0])
|
||||
if data[names_off + section[0]:name_end] == b".comment":
|
||||
raw = data[section[4]:section[4] + section[5]]
|
||||
return sorted({item.decode("utf-8", "replace") for item in raw.split(b"\0") if item})
|
||||
return []
|
||||
|
||||
|
||||
def mednafen_state_vars(state: bytes) -> dict[str, str]:
|
||||
"""SECTION.VAR -> hex value from a Mednafen MDFNSVST save state."""
|
||||
start = state.find(b"MDFNSVST")
|
||||
if start < 0:
|
||||
return {}
|
||||
pos = start + 32
|
||||
out: dict[str, str] = {}
|
||||
while pos + 36 <= len(state):
|
||||
section = state[pos:pos + 32].split(b"\0", 1)[0].decode("ascii", "replace")
|
||||
size = struct.unpack_from("<I", state, pos + 32)[0]
|
||||
body = state[pos + 36:pos + 36 + size]
|
||||
pos += 36 + size
|
||||
cursor = 0
|
||||
while cursor < len(body):
|
||||
name_len = body[cursor]
|
||||
name = body[cursor + 1:cursor + 1 + name_len].decode("ascii", "replace")
|
||||
cursor += 1 + name_len
|
||||
if cursor + 4 > len(body):
|
||||
break
|
||||
value_len = struct.unpack_from("<I", body, cursor)[0] & 0x7FFFFFFF
|
||||
cursor += 4
|
||||
out[f"{section}.{name}"] = body[cursor:cursor + value_len].hex()
|
||||
cursor += value_len
|
||||
return out
|
||||
|
||||
|
||||
def render_printf(fmt: bytes, args: Sequence[int]) -> str:
|
||||
"""Render a C format string with the integer-class arguments captured.
|
||||
|
||||
Floating-point arguments travel in vector registers, which a ctypes
|
||||
callback cannot see: they are shown as <double>.
|
||||
"""
|
||||
text = fmt.decode("utf-8", "replace")
|
||||
queue = list(args)
|
||||
out: list[str] = []
|
||||
last = 0
|
||||
for match in PRINTF_SPEC.finditer(text):
|
||||
out.append(text[last:match.start()])
|
||||
last = match.end()
|
||||
conv = match.group("conv")
|
||||
if conv == "%":
|
||||
out.append("%")
|
||||
continue
|
||||
for star in (match.group("width"), match.group("prec")):
|
||||
if star == "*" and queue:
|
||||
queue.pop(0)
|
||||
if conv in "eEfFgGaA":
|
||||
out.append("<double>")
|
||||
continue
|
||||
value = queue.pop(0) if queue else 0
|
||||
if conv == "s":
|
||||
out.append(ctypes.string_at(value).decode("utf-8", "replace") if value else "(null)")
|
||||
elif conv == "c":
|
||||
out.append(chr(value & 0xFF))
|
||||
elif conv == "p":
|
||||
out.append(hex(value))
|
||||
elif conv in "di":
|
||||
bits = 64 if match.group("len") in ("l", "ll", "j", "z", "t") else 32
|
||||
value &= (1 << bits) - 1
|
||||
out.append(str(value - (1 << bits) if value >> (bits - 1) else value))
|
||||
else:
|
||||
bits = 64 if match.group("len") in ("l", "ll", "j", "z", "t") else 32
|
||||
value &= (1 << bits) - 1
|
||||
out.append({"o": f"{value:o}", "x": f"{value:x}", "X": f"{value:X}"}.get(conv, str(value)))
|
||||
out.append(text[last:])
|
||||
return "".join(out).rstrip("\n")
|
||||
|
||||
|
||||
def png_bytes(width: int, height: int, rgb: bytes) -> bytes:
|
||||
def chunk(tag: bytes, body: bytes) -> bytes:
|
||||
return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body))
|
||||
rows = b"".join(b"\0" + rgb[y * width * 3:(y + 1) * width * 3] for y in range(height))
|
||||
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(rows, 6)) + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
def to_rgb(fmt: int, width: int, height: int, packed: bytes) -> bytes:
|
||||
out = bytearray(width * height * 3)
|
||||
if fmt == 1:
|
||||
out[0::3], out[1::3], out[2::3] = packed[2::4], packed[1::4], packed[0::4]
|
||||
return bytes(out)
|
||||
for i, (pixel,) in enumerate(struct.iter_unpack("<H", packed)):
|
||||
if fmt == 2:
|
||||
r, g, b = (pixel >> 11) & 31, (pixel >> 5) & 63, pixel & 31
|
||||
out[i * 3:i * 3 + 3] = bytes((r << 3, g << 2, b << 3))
|
||||
else:
|
||||
r, g, b = (pixel >> 10) & 31, (pixel >> 5) & 31, pixel & 31
|
||||
out[i * 3:i * 3 + 3] = bytes((r << 3, g << 3, b << 3))
|
||||
return bytes(out)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# libretro bindings (worker side)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class RetroSystemInfo(ctypes.Structure):
|
||||
_fields_ = [("library_name", ctypes.c_char_p), ("library_version", ctypes.c_char_p),
|
||||
("valid_extensions", ctypes.c_char_p), ("need_fullpath", ctypes.c_bool),
|
||||
("block_extract", ctypes.c_bool)]
|
||||
|
||||
|
||||
class RetroGameGeometry(ctypes.Structure):
|
||||
_fields_ = [("base_width", ctypes.c_uint), ("base_height", ctypes.c_uint),
|
||||
("max_width", ctypes.c_uint), ("max_height", ctypes.c_uint),
|
||||
("aspect_ratio", ctypes.c_float)]
|
||||
|
||||
|
||||
class RetroSystemTiming(ctypes.Structure):
|
||||
_fields_ = [("fps", ctypes.c_double), ("sample_rate", ctypes.c_double)]
|
||||
|
||||
|
||||
class RetroSystemAvInfo(ctypes.Structure):
|
||||
_fields_ = [("geometry", RetroGameGeometry), ("timing", RetroSystemTiming)]
|
||||
|
||||
|
||||
class RetroGameInfo(ctypes.Structure):
|
||||
_fields_ = [("path", ctypes.c_char_p), ("data", ctypes.c_void_p),
|
||||
("size", ctypes.c_size_t), ("meta", ctypes.c_char_p)]
|
||||
|
||||
|
||||
class RetroVariable(ctypes.Structure):
|
||||
_fields_ = [("key", ctypes.c_char_p), ("value", ctypes.c_char_p)]
|
||||
|
||||
|
||||
LOG_ARGS = 16
|
||||
LogFn = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p, *([ctypes.c_size_t] * LOG_ARGS))
|
||||
|
||||
|
||||
class RetroLogCallback(ctypes.Structure):
|
||||
_fields_ = [("log", LogFn)]
|
||||
|
||||
|
||||
EnvFn = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_uint, ctypes.c_void_p)
|
||||
VideoFn = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_uint, ctypes.c_uint, ctypes.c_size_t)
|
||||
AudioFn = ctypes.CFUNCTYPE(None, ctypes.c_int16, ctypes.c_int16)
|
||||
AudioBatchFn = ctypes.CFUNCTYPE(ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t)
|
||||
PollFn = ctypes.CFUNCTYPE(None)
|
||||
InputFn = ctypes.CFUNCTYPE(ctypes.c_int16, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint)
|
||||
|
||||
|
||||
class Frontend:
|
||||
"""The libretro callbacks for one run."""
|
||||
|
||||
def __init__(self, spec: dict) -> None:
|
||||
self.spec = spec
|
||||
self.system_dir = spec["system_dir"].encode()
|
||||
self.save_dir = spec["save_dir"].encode()
|
||||
self.options: dict[bytes, bytes] = {k.encode(): v.encode() for k, v in spec["options"].items()}
|
||||
self.defaults: dict[bytes, bytes] = {}
|
||||
self.declared: dict[str, list[str]] = {}
|
||||
self.option_buffers: list[ctypes.Array] = []
|
||||
self.pixel_format = 0
|
||||
self.frame = 0
|
||||
self.last_frame = b""
|
||||
self.last_size = (0, 0)
|
||||
self.video = hashlib.sha1()
|
||||
self.audio = hashlib.sha1()
|
||||
self.frame_hashes: list[list] = []
|
||||
self.logs: list[dict] = []
|
||||
self.presses: list[tuple[int, int, int]] = [tuple(p) for p in spec["presses"]]
|
||||
self.unanswered: dict[int, int] = {}
|
||||
self.input_queries = 0
|
||||
self.input_pressed = 0
|
||||
self.callbacks = (
|
||||
EnvFn(self.environment), VideoFn(self.video_refresh), AudioFn(self.audio_sample),
|
||||
AudioBatchFn(self.audio_batch), PollFn(lambda: None), InputFn(self.input_state),
|
||||
)
|
||||
self.log_callback = RetroLogCallback(LogFn(self.log))
|
||||
|
||||
def log(self, level: int, fmt: bytes, *args: int) -> None:
|
||||
name = LOG_LEVELS[level] if 0 <= level < len(LOG_LEVELS) else str(level)
|
||||
self.logs.append({"frame": self.frame, "level": name, "text": render_printf(fmt or b"", args)})
|
||||
|
||||
def environment(self, cmd: int, data: int) -> bool:
|
||||
cmd &= ~ENV_EXPERIMENTAL
|
||||
if cmd == ENV_GET_CAN_DUPE:
|
||||
ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] = True
|
||||
return True
|
||||
if cmd in (ENV_GET_SYSTEM_DIRECTORY, ENV_GET_SAVE_DIRECTORY):
|
||||
value = self.system_dir if cmd == ENV_GET_SYSTEM_DIRECTORY else self.save_dir
|
||||
buffer = ctypes.create_string_buffer(value)
|
||||
self.option_buffers.append(buffer)
|
||||
ctypes.cast(data, ctypes.POINTER(ctypes.c_char_p))[0] = ctypes.cast(buffer, ctypes.c_char_p)
|
||||
return True
|
||||
if cmd == ENV_SET_PIXEL_FORMAT:
|
||||
fmt = ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0]
|
||||
if fmt not in PIXEL_BYTES:
|
||||
return False
|
||||
self.pixel_format = fmt
|
||||
return True
|
||||
if cmd == ENV_SET_VARIABLES:
|
||||
array = ctypes.cast(data, ctypes.POINTER(RetroVariable))
|
||||
index = 0
|
||||
while array[index].key:
|
||||
key, desc = array[index].key, (array[index].value or b"").decode("utf-8", "replace")
|
||||
values = desc.split(";", 1)[-1].strip().split("|")
|
||||
self.declared[key.decode()] = values
|
||||
self.defaults[key] = values[0].encode()
|
||||
index += 1
|
||||
return True
|
||||
if cmd == ENV_GET_VARIABLE:
|
||||
var = ctypes.cast(data, ctypes.POINTER(RetroVariable))[0]
|
||||
value = self.options.get(var.key, self.defaults.get(var.key))
|
||||
if value is None:
|
||||
return False
|
||||
buffer = ctypes.create_string_buffer(value)
|
||||
self.option_buffers.append(buffer)
|
||||
ctypes.cast(data, ctypes.POINTER(RetroVariable))[0].value = ctypes.cast(buffer, ctypes.c_char_p)
|
||||
return True
|
||||
if cmd == ENV_GET_VARIABLE_UPDATE:
|
||||
ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] = False
|
||||
return True
|
||||
if cmd == ENV_SET_SUPPORT_NO_GAME:
|
||||
return True
|
||||
if cmd == ENV_GET_LOG_INTERFACE:
|
||||
ctypes.cast(data, ctypes.POINTER(RetroLogCallback))[0] = self.log_callback
|
||||
return True
|
||||
if cmd == ENV_GET_CORE_OPTIONS_VERSION:
|
||||
ctypes.cast(data, ctypes.POINTER(ctypes.c_uint))[0] = 0
|
||||
return True
|
||||
self.unanswered[cmd] = self.unanswered.get(cmd, 0) + 1
|
||||
return False
|
||||
|
||||
def video_refresh(self, data: int, width: int, height: int, pitch: int) -> None:
|
||||
if data and data != ctypes.c_void_p(-1).value:
|
||||
row = width * PIXEL_BYTES[self.pixel_format]
|
||||
raw = ctypes.string_at(data, pitch * (height - 1) + row) if height else b""
|
||||
self.last_frame = b"".join(raw[y * pitch:y * pitch + row] for y in range(height))
|
||||
self.last_size = (width, height)
|
||||
self.video.update(struct.pack("<II", *self.last_size) + hashlib.sha1(self.last_frame).digest())
|
||||
every = self.spec["frame_hash_every"]
|
||||
if every and self.frame % every == 0:
|
||||
self.frame_hashes.append([self.frame, hashlib.sha1(self.last_frame).hexdigest()])
|
||||
png_dir = self.spec.get("png_dir")
|
||||
if png_dir and self.last_frame:
|
||||
rgb = to_rgb(self.pixel_format, *self.last_size, self.last_frame)
|
||||
Path(png_dir, f"frame{self.frame:07d}.png").write_bytes(png_bytes(*self.last_size, rgb))
|
||||
|
||||
def audio_sample(self, left: int, right: int) -> None:
|
||||
self.audio.update(struct.pack("<hh", left, right))
|
||||
|
||||
def audio_batch(self, data: int, frames: int) -> int:
|
||||
self.audio.update(ctypes.string_at(data, frames * 4))
|
||||
return frames
|
||||
|
||||
def input_state(self, port: int, device: int, index: int, button: int) -> int:
|
||||
self.input_queries += 1
|
||||
if port or device != 1 or index:
|
||||
return 0
|
||||
held = int(any(start <= self.frame < start + length and button == pressed
|
||||
for start, pressed, length in self.presses))
|
||||
self.input_pressed += held
|
||||
return held
|
||||
|
||||
|
||||
def run_worker(spec: dict) -> dict:
|
||||
"""Load the core in this process and run it. Returns the measured result."""
|
||||
core = ctypes.CDLL(spec["core"])
|
||||
front = Frontend(spec)
|
||||
for name, fn in zip(("retro_set_environment", "retro_set_video_refresh", "retro_set_audio_sample",
|
||||
"retro_set_audio_sample_batch", "retro_set_input_poll", "retro_set_input_state"),
|
||||
front.callbacks):
|
||||
getattr(core, name)(fn)
|
||||
info = RetroSystemInfo()
|
||||
core.retro_get_system_info(ctypes.byref(info))
|
||||
core.retro_init()
|
||||
game = None
|
||||
if spec.get("content"):
|
||||
path = spec["content"].encode()
|
||||
game = RetroGameInfo(path, None, 0, None)
|
||||
if not info.need_fullpath:
|
||||
blob = Path(spec["content"]).read_bytes()
|
||||
holder = ctypes.create_string_buffer(blob, len(blob))
|
||||
front.option_buffers.append(holder)
|
||||
game.data, game.size = ctypes.cast(holder, ctypes.c_void_p), len(blob)
|
||||
core.retro_load_game.restype = ctypes.c_bool
|
||||
if not core.retro_load_game(ctypes.byref(game) if game else None):
|
||||
return {"loaded": False, "logs": front.logs, "library_name": info.library_name.decode()}
|
||||
# A frontend plugs a joypad into each port after loading, as RetroArch does.
|
||||
for port in range(spec["joypad_ports"]):
|
||||
core.retro_set_controller_port_device(ctypes.c_uint(port), ctypes.c_uint(1))
|
||||
av = RetroSystemAvInfo()
|
||||
core.retro_get_system_av_info(ctypes.byref(av))
|
||||
warmup, frames = spec["warmup"], spec["frames"]
|
||||
for _ in range(warmup):
|
||||
core.retro_run()
|
||||
front.frame += 1
|
||||
started = time.perf_counter()
|
||||
for _ in range(frames):
|
||||
core.retro_run()
|
||||
front.frame += 1
|
||||
wall = time.perf_counter() - started
|
||||
core.retro_serialize_size.restype = ctypes.c_size_t
|
||||
size = core.retro_serialize_size()
|
||||
state = b""
|
||||
if size:
|
||||
buffer = ctypes.create_string_buffer(size)
|
||||
core.retro_serialize.restype = ctypes.c_bool
|
||||
if core.retro_serialize(buffer, ctypes.c_size_t(size)):
|
||||
state = buffer.raw
|
||||
if spec.get("state_out"):
|
||||
Path(spec["state_out"]).write_bytes(state)
|
||||
state_vars = mednafen_state_vars(state)
|
||||
observables = {
|
||||
"frame_sha1": hashlib.sha1(front.last_frame).hexdigest(),
|
||||
"video_sha1": front.video.hexdigest(),
|
||||
"audio_sha1": front.audio.hexdigest(),
|
||||
"state_sha1": hashlib.sha1(state).hexdigest() if state else None,
|
||||
}
|
||||
for name in spec["state_vars"]:
|
||||
observables[f"state:{name}"] = state_vars.get(name)
|
||||
if spec["frame_hash_every"]:
|
||||
observables["frame_hashes"] = front.frame_hashes
|
||||
core.retro_unload_game()
|
||||
core.retro_deinit()
|
||||
return {
|
||||
"loaded": True,
|
||||
"library_name": info.library_name.decode(),
|
||||
"library_version": info.library_version.decode(),
|
||||
"fps_nominal": av.timing.fps,
|
||||
"frames": frames,
|
||||
"warmup": warmup,
|
||||
"wall_s": wall,
|
||||
"observables": observables,
|
||||
"logs": front.logs,
|
||||
"options_declared": front.declared,
|
||||
"env_unanswered": {str(k): v for k, v in sorted(front.unanswered.items())},
|
||||
"last_frame_size": list(front.last_size),
|
||||
"input_queries": front.input_queries,
|
||||
"input_pressed": front.input_pressed,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Host, core and frontend identity (parent side)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def load_average() -> list[float] | None:
|
||||
try:
|
||||
return [round(value, 2) for value in os.getloadavg()]
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def cpu_model() -> str | None:
|
||||
try:
|
||||
for line in Path("/proc/cpuinfo").read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("model name"):
|
||||
return line.split(":", 1)[1].strip()
|
||||
except OSError:
|
||||
return platform.processor() or None
|
||||
return platform.processor() or None
|
||||
|
||||
|
||||
def host_identity() -> dict:
|
||||
return {
|
||||
"cpu_model": cpu_model(),
|
||||
"cpu_count": os.cpu_count(),
|
||||
"os": platform.platform(),
|
||||
"kernel": platform.release(),
|
||||
"machine": platform.machine(),
|
||||
"python": platform.python_version(),
|
||||
}
|
||||
|
||||
|
||||
def frontend_identity() -> dict:
|
||||
script = Path(__file__).resolve()
|
||||
commit: str | None = None
|
||||
dirty: bool | None = None
|
||||
try:
|
||||
commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True,
|
||||
text=True, check=True).stdout.strip()
|
||||
status = subprocess.run(["git", "status", "--porcelain", "--", str(script)], cwd=ROOT,
|
||||
capture_output=True, text=True, check=True).stdout
|
||||
dirty = bool(status.strip())
|
||||
except (OSError, subprocess.CalledProcessError) as exc:
|
||||
print(f"WARNING: frontend commit unknown, git failed: {exc}", file=sys.stderr)
|
||||
return {"script": "scripts/core_probe.py", "commit": commit, "script_modified": dirty,
|
||||
"script_sha256": file_digest(script, "sha256")}
|
||||
|
||||
|
||||
def core_identity(args: argparse.Namespace) -> dict:
|
||||
core = Path(args.core)
|
||||
return {
|
||||
"file": core.name,
|
||||
"sha256": file_digest(core, "sha256"),
|
||||
"repo": args.core_repo,
|
||||
"commit": args.core_commit,
|
||||
"build_command": args.build_command,
|
||||
"compiler": elf_comment(core) or None,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Runs (parent side)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class Variant:
|
||||
label: str
|
||||
files: dict[str, Path | None] = field(default_factory=dict)
|
||||
|
||||
|
||||
def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> dict[str, str | None]:
|
||||
"""Copy the system files of one variant. Returns dest -> sha1 (None = absent)."""
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
staged: dict[str, str | None] = {}
|
||||
for dest, src in {**base, **variant.files}.items():
|
||||
out = target / dest
|
||||
if src is None:
|
||||
staged[dest] = None
|
||||
continue
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(src, out)
|
||||
staged[dest] = file_digest(out)
|
||||
return staged
|
||||
|
||||
|
||||
def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None,
|
||||
png_dir: Path | None) -> dict:
|
||||
save_dir = scratch / "save"
|
||||
save_dir.mkdir(parents=True, exist_ok=True)
|
||||
spec = {
|
||||
"core": str(Path(args.core).resolve()),
|
||||
"content": str(Path(args.content).resolve()) if args.content else None,
|
||||
"system_dir": str(system_dir.resolve()),
|
||||
"save_dir": str(save_dir.resolve()),
|
||||
"options": dict(args.option),
|
||||
"frames": args.frames,
|
||||
"warmup": args.warmup,
|
||||
"presses": [list(p) for p in args.press],
|
||||
"state_vars": args.state_var,
|
||||
"frame_hash_every": args.frame_hash_every,
|
||||
"joypad_ports": args.joypad_ports,
|
||||
"png_dir": str(png_dir) if png_dir else None,
|
||||
"state_out": str(state_out) if state_out else None,
|
||||
}
|
||||
spec_path, out_path, trace_path = scratch / "spec.json", scratch / "result.json", scratch / "strace.txt"
|
||||
spec_path.write_text(json.dumps(spec), encoding="utf-8")
|
||||
command = [sys.executable, "-I", str(Path(__file__).resolve()), "_worker", str(spec_path), str(out_path)]
|
||||
strace = shutil.which("strace") if not args.no_trace else None
|
||||
if strace:
|
||||
command = [strace, "-f", "-qq", "-s", "4096", "-e", "trace=openat,open", "-o", str(trace_path)] + command
|
||||
env = {**os.environ, **dict(args.env)}
|
||||
proc = subprocess.run(command, check=False, capture_output=True, text=True,
|
||||
timeout=args.timeout, env=env)
|
||||
if proc.returncode != 0 or not out_path.exists():
|
||||
raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}")
|
||||
result = json.loads(out_path.read_text(encoding="utf-8"))
|
||||
if strace:
|
||||
opened, probed = files_under(parse_strace(trace_path.read_text(encoding="utf-8", errors="replace")),
|
||||
system_dir)
|
||||
result["files_opened"] = [{"path": rel, "sha1_after": file_digest(system_dir / rel)}
|
||||
for rel in opened if (system_dir / rel).is_file()]
|
||||
result["files_probed_absent"] = probed
|
||||
result["tracing"] = "strace"
|
||||
else:
|
||||
result["files_opened"] = None
|
||||
result["files_probed_absent"] = None
|
||||
result["tracing"] = "unavailable: strace not found" if not args.no_trace else "disabled: --no-trace"
|
||||
if result.get("wall_s") and result.get("frames"):
|
||||
result["ms_per_frame"] = result["wall_s"] * 1000 / result["frames"]
|
||||
result["fps"] = result["frames"] / result["wall_s"]
|
||||
return result
|
||||
|
||||
|
||||
def timing_stats(values: Sequence[float]) -> dict:
|
||||
return {
|
||||
"median": statistics.median(values),
|
||||
"mean": statistics.fmean(values),
|
||||
"stdev": statistics.stdev(values) if len(values) > 1 else 0.0,
|
||||
"min": min(values),
|
||||
"max": max(values),
|
||||
"raw": list(values),
|
||||
}
|
||||
|
||||
|
||||
def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Variant,
|
||||
work: Path, repetitions: int) -> dict:
|
||||
system_dir = work / "system"
|
||||
staged = stage_system_dir(base, variant, system_dir)
|
||||
reps: list[dict] = []
|
||||
flags: list[str] = []
|
||||
for index in range(repetitions):
|
||||
scratch = work / f"rep{index}"
|
||||
scratch.mkdir(parents=True, exist_ok=True)
|
||||
png_dir = None
|
||||
if args.png_dir and index == 0:
|
||||
png_dir = Path(args.png_dir) / variant.label
|
||||
png_dir.mkdir(parents=True, exist_ok=True)
|
||||
state_out = Path(args.state_dir) / f"{variant.label}.state" if args.state_dir and index == 0 else None
|
||||
if state_out:
|
||||
state_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
rep = run_once(args, system_dir, scratch, state_out, png_dir)
|
||||
if not rep.get("loaded"):
|
||||
flags.append(f"repetition {index}: retro_load_game returned false")
|
||||
for entry in rep.get("files_opened") or []:
|
||||
expected = staged.get(entry["path"])
|
||||
entry["sha1_staged"] = expected
|
||||
if expected is not None and expected != entry["sha1_after"]:
|
||||
flags.append(f"repetition {index}: {entry['path']} sha1 changed during the run")
|
||||
reps.append(rep)
|
||||
reference = reps[0].get("observables")
|
||||
for index, rep in enumerate(reps[1:], 1):
|
||||
if rep.get("observables") != reference:
|
||||
differing = sorted(k for k in (reference or {}) if (rep.get("observables") or {}).get(k) != reference[k])
|
||||
flags.append(f"repetition {index} disagrees with repetition 0 on {', '.join(differing)}")
|
||||
return {
|
||||
"label": variant.label,
|
||||
"system_files": {dest: sha for dest, sha in sorted(staged.items())},
|
||||
"repetitions": reps,
|
||||
"flags": flags,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Command line
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
fromfile_prefix_chars="@")
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
common = argparse.ArgumentParser(add_help=False, fromfile_prefix_chars="@")
|
||||
common.add_argument("--core", required=True, help="libretro .so to load")
|
||||
common.add_argument("--content", help="content to load (a .cue is identified with its tracks)")
|
||||
common.add_argument("--content-label", help="identity of the content, e.g. its Redump name")
|
||||
common.add_argument("--system-file", action="append", default=[], metavar="DEST=SRC",
|
||||
type=lambda t: parse_assignment(t, "--system-file"),
|
||||
help="copy SRC into the staged system dir as DEST")
|
||||
common.add_argument("--option", action="append", default=[], metavar="KEY=VALUE",
|
||||
type=lambda t: parse_assignment(t, "--option"), help="core option")
|
||||
common.add_argument("--env", action="append", default=[], metavar="KEY=VALUE",
|
||||
type=lambda t: parse_assignment(t, "--env"),
|
||||
help="environment variable for the core process, e.g. LD_PRELOAD of a fixed clock")
|
||||
common.add_argument("--frames", type=int, default=600, help="frames timed after warm-up")
|
||||
common.add_argument("--warmup", type=int, default=0, help="frames run before timing starts")
|
||||
common.add_argument("--press", action="append", default=[], type=parse_press,
|
||||
metavar="FRAME:BUTTON[:LEN]", help="hold a joypad button on port 0")
|
||||
common.add_argument("--joypad-ports", type=int, default=2, metavar="N",
|
||||
help="ports given a joypad after loading (default 2)")
|
||||
common.add_argument("--state-var", action="append", default=[], metavar="SECTION.VAR",
|
||||
help="report a Mednafen save-state variable as an observable")
|
||||
common.add_argument("--frame-hash-every", type=int, default=0, metavar="K",
|
||||
help="record the sha1 of every K-th frame")
|
||||
common.add_argument("--png-dir", help="write the frames sampled by --frame-hash-every as PNG")
|
||||
common.add_argument("--state-dir", help="write the save state at the end of the first repetition")
|
||||
common.add_argument("--core-repo", help="source repository URL of the core build")
|
||||
common.add_argument("--core-commit", help="commit the core was built from")
|
||||
common.add_argument("--build-command", help="command and flags that built the core")
|
||||
common.add_argument("--record", metavar="DIR", help="write one JSON record under DIR/<core>/")
|
||||
common.add_argument("--max-load", type=float, default=None,
|
||||
help="refuse to record when the 1-minute load average exceeds this "
|
||||
"(default: a quarter of the CPU count)")
|
||||
common.add_argument("--work-dir", default=str(DEFAULT_WORK_DIR), help="scratch directory")
|
||||
common.add_argument("--timeout", type=int, default=1800, help="seconds per repetition")
|
||||
common.add_argument("--no-trace", action="store_true", help="do not run the worker under strace")
|
||||
|
||||
sub.add_parser("run", parents=[common], help="one execution")
|
||||
bench = sub.add_parser("bench", parents=[common], help="timed repetitions")
|
||||
bench.add_argument("--repeat", type=int, default=MIN_BENCH_REPETITIONS)
|
||||
check = sub.add_parser("check", parents=[common], help="compare variants of a system file")
|
||||
check.add_argument("--vary", action="append", required=True, metavar="DEST=SRC",
|
||||
type=lambda t: parse_assignment(t, "--vary"),
|
||||
help="one variant per occurrence; an empty SRC leaves DEST absent")
|
||||
check.add_argument("--observable", required=True,
|
||||
help="the observable compared across variants")
|
||||
check.add_argument("--claim", required=True, help="the statement under test")
|
||||
check.add_argument("--method", required=True, help="how the run tests it")
|
||||
check.add_argument("--repeat", type=int, default=2)
|
||||
worker = sub.add_parser("_worker")
|
||||
worker.add_argument("spec")
|
||||
worker.add_argument("out")
|
||||
return parser
|
||||
|
||||
|
||||
def validate_args(args: argparse.Namespace) -> list[str]:
|
||||
problems: list[str] = []
|
||||
if not Path(args.core).is_file():
|
||||
problems.append(f"--core {args.core}: not a file")
|
||||
if args.content and not Path(args.content).is_file():
|
||||
problems.append(f"--content {args.content}: not a file")
|
||||
for dest, src in args.system_file:
|
||||
if not Path(src).is_file():
|
||||
problems.append(f"--system-file {dest}={src}: not a file")
|
||||
if args.frames < 1 or args.warmup < 0:
|
||||
problems.append("--frames must be >= 1 and --warmup >= 0")
|
||||
if args.png_dir and not args.frame_hash_every:
|
||||
problems.append("--png-dir needs --frame-hash-every")
|
||||
if args.mode == "bench" and args.record and args.repeat < MIN_BENCH_REPETITIONS:
|
||||
problems.append(f"a recorded benchmark needs --repeat >= {MIN_BENCH_REPETITIONS}")
|
||||
if args.mode == "check":
|
||||
dests = {dest for dest, _ in args.vary}
|
||||
if len(dests) != 1:
|
||||
problems.append("--vary must name the same DEST in every occurrence")
|
||||
for dest, src in args.vary:
|
||||
if src and not Path(src).is_file():
|
||||
problems.append(f"--vary {dest}={src}: not a file")
|
||||
names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var}
|
||||
if args.frame_hash_every:
|
||||
names.add("frame_hashes")
|
||||
if args.observable not in names:
|
||||
problems.append(f"--observable {args.observable}: not one of {', '.join(sorted(names))}")
|
||||
return problems
|
||||
|
||||
|
||||
def variants_for(args: argparse.Namespace) -> list[Variant]:
|
||||
if args.mode != "check":
|
||||
return [Variant("default")]
|
||||
out: list[Variant] = []
|
||||
seen: set[str] = set()
|
||||
for dest, src in args.vary:
|
||||
label = Path(src).name if src else f"{dest}-absent"
|
||||
while label in seen:
|
||||
label += "+"
|
||||
seen.add(label)
|
||||
out.append(Variant(label, {dest: Path(src) if src else None}))
|
||||
return out
|
||||
|
||||
|
||||
def check_result(variants: list[dict], observable: str) -> dict:
|
||||
groups: dict[str, list[str]] = {}
|
||||
failed = [v["label"] for v in variants if v["flags"]]
|
||||
for variant in variants:
|
||||
value = (variant["repetitions"][0].get("observables") or {}).get(observable)
|
||||
groups.setdefault(json.dumps(value, sort_keys=True), []).append(variant["label"])
|
||||
if failed:
|
||||
status = "inconclusive"
|
||||
statement = (f"{len(failed)} of {len(variants)} variants carry flags "
|
||||
f"({', '.join(failed)}); no comparison is drawn from them")
|
||||
elif len(groups) == 1:
|
||||
status = "same"
|
||||
statement = f"{observable} is identical across all {len(variants)} variants"
|
||||
else:
|
||||
status = "differs"
|
||||
statement = f"{observable} takes {len(groups)} distinct values across {len(variants)} variants"
|
||||
return {
|
||||
"status": status,
|
||||
"statement": statement,
|
||||
"groups": [{"value": json.loads(key), "variants": labels} for key, labels in groups.items()],
|
||||
}
|
||||
|
||||
|
||||
def record_path(directory: Path, core_name: str, record: dict) -> Path:
|
||||
canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
|
||||
digest = hashlib.sha256(canonical).hexdigest()[:12]
|
||||
date = record["recorded_at"][:10]
|
||||
slug = re.sub(r"[^a-z0-9]+", "_", core_name.lower()).strip("_") or "core"
|
||||
return directory / slug / f"{date}-{digest}.json"
|
||||
|
||||
|
||||
def probe(args: argparse.Namespace) -> int:
|
||||
problems = validate_args(args)
|
||||
if problems:
|
||||
for problem in problems:
|
||||
print(f"ERROR: {problem}", file=sys.stderr)
|
||||
return 2
|
||||
max_load = args.max_load if args.max_load is not None else (os.cpu_count() or 1) / 4
|
||||
load_before = load_average()
|
||||
if args.record and load_before and load_before[0] > max_load:
|
||||
print(f"ERROR: load average {load_before[0]} exceeds {max_load}: the machine is busy, "
|
||||
f"nothing recorded (raise --max-load to override)", file=sys.stderr)
|
||||
return 3
|
||||
repetitions = 1 if args.mode == "run" else args.repeat
|
||||
base = {dest: Path(src) for dest, src in args.system_file}
|
||||
work = Path(tempfile.mkdtemp(prefix=f"{args.mode}-", dir=_ensure(Path(args.work_dir))))
|
||||
try:
|
||||
variants = [run_variant(args, base, variant, work / f"v{i}", repetitions)
|
||||
for i, variant in enumerate(variants_for(args))]
|
||||
except (ProbeError, subprocess.TimeoutExpired) as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
load_after = load_average()
|
||||
first = variants[0]["repetitions"][0]
|
||||
record: dict = {
|
||||
"record_version": RECORD_VERSION,
|
||||
"kind": args.mode,
|
||||
"recorded_at": dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat(),
|
||||
"core": {**core_identity(args), "library_name": first.get("library_name"),
|
||||
"library_version": first.get("library_version")},
|
||||
"frontend": frontend_identity(),
|
||||
"host": {**host_identity(), "load_before": load_before, "load_after": load_after,
|
||||
"max_load": max_load},
|
||||
"inputs": {
|
||||
"content": content_identity(Path(args.content), args.content_label) if args.content else None,
|
||||
"options": dict(args.option),
|
||||
"env": dict(args.env),
|
||||
"frames": args.frames,
|
||||
"warmup": args.warmup,
|
||||
"repetitions": repetitions,
|
||||
"presses": [{"frame": f, "button": b, "length": n} for f, b, n in args.press],
|
||||
"observables": list(BASE_OBSERVABLES) + [f"state:{v}" for v in args.state_var]
|
||||
+ (["frame_hashes"] if args.frame_hash_every else []),
|
||||
"frame_hash_every": args.frame_hash_every,
|
||||
"joypad_ports": args.joypad_ports,
|
||||
},
|
||||
"variants": variants,
|
||||
"flags": [f"{v['label']}: {flag}" for v in variants for flag in v["flags"]],
|
||||
}
|
||||
if load_before and load_after and max(load_before[0], load_after[0]) > max_load:
|
||||
record["flags"].append(f"load average reached {max(load_before[0], load_after[0])} "
|
||||
f"(threshold {max_load})")
|
||||
if args.mode == "bench":
|
||||
reps = variants[0]["repetitions"]
|
||||
record["timing"] = {"ms_per_frame": timing_stats([r["ms_per_frame"] for r in reps if "ms_per_frame" in r]),
|
||||
"fps": timing_stats([r["fps"] for r in reps if "fps" in r])}
|
||||
if args.mode == "check":
|
||||
record["check"] = {"claim": args.claim, "method": args.method, "observable": args.observable,
|
||||
"variable": args.vary[0][0],
|
||||
"result": check_result(variants, args.observable)}
|
||||
text = json.dumps(record, indent=2, sort_keys=False)
|
||||
if args.record:
|
||||
path = record_path(Path(args.record), first.get("library_name") or Path(args.core).stem, record)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text + "\n", encoding="utf-8")
|
||||
print(f"recorded {path}", file=sys.stderr)
|
||||
print(text)
|
||||
return 0
|
||||
|
||||
|
||||
def _ensure(path: Path) -> Path:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
if args.mode == "_worker":
|
||||
spec = json.loads(Path(args.spec).read_text(encoding="utf-8"))
|
||||
Path(args.out).write_text(json.dumps(run_worker(spec)), encoding="utf-8")
|
||||
return 0
|
||||
return probe(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -616,6 +616,12 @@ def main() -> int:
|
||||
errors.extend(_save_directory_blocks())
|
||||
errors.extend(_registry_contract())
|
||||
errors.extend(_event_schema_errors())
|
||||
errors.extend(
|
||||
_validate_json_files(
|
||||
sorted((ROOT / "measurements").glob("**/*.json")),
|
||||
"core-probe-record.schema.json",
|
||||
)
|
||||
)
|
||||
|
||||
if not args.source_only:
|
||||
database_path = ROOT / "database.json"
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
"""core_probe runs a libretro core headless and records what it measured.
|
||||
|
||||
The parsers are checked on fixed inputs. One test builds a ten-line
|
||||
libretro core with the C compiler, when there is one, and runs the whole
|
||||
probe on it: staging, the worker process, strace, the record and its schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
import core_probe
|
||||
|
||||
STRACE_SAMPLE = """\
|
||||
4242 openat(AT_FDCWD, "/sys/bios/card.bin", O_RDONLY) = 5
|
||||
4242 openat(AT_FDCWD, "/sys/bios/missing.bin", O_RDONLY) = -1 ENOENT (No such file or directory)
|
||||
[pid 4243] open("/sys/bios/sub/x.rom", O_RDONLY|O_CLOEXEC) = 7
|
||||
4242 openat(AT_FDCWD, "/elsewhere/game.cue", O_RDONLY) = 3
|
||||
4242 read(3, "...", 10) = 10
|
||||
"""
|
||||
|
||||
FAKE_CORE = r"""
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
struct retro_system_info { const char *n, *v, *e; bool full, block; };
|
||||
struct retro_game_geometry { unsigned bw, bh, mw, mh; float a; };
|
||||
struct retro_system_timing { double fps, rate; };
|
||||
struct retro_system_av_info { struct retro_game_geometry g; struct retro_system_timing t; };
|
||||
struct retro_log_callback { void (*log)(int, const char *, ...); };
|
||||
typedef bool (*env_t)(unsigned, void *);
|
||||
typedef void (*video_t)(const void *, unsigned, unsigned, size_t);
|
||||
typedef size_t (*batch_t)(const int16_t *, size_t);
|
||||
typedef int16_t (*input_t)(unsigned, unsigned, unsigned, unsigned);
|
||||
static env_t env; static video_t video; static batch_t batch; static input_t input;
|
||||
static unsigned ports, frames; static struct retro_log_callback logger;
|
||||
static uint16_t pixels[4 * 2]; static unsigned char card;
|
||||
void retro_set_environment(env_t e) { bool t = true; env = e; env(18, &t); }
|
||||
void retro_set_video_refresh(video_t v) { video = v; }
|
||||
void retro_set_audio_sample(void *f) { (void)f; }
|
||||
void retro_set_audio_sample_batch(batch_t b) { batch = b; }
|
||||
void retro_set_input_poll(void *f) { (void)f; }
|
||||
void retro_set_input_state(input_t f) { input = f; }
|
||||
void retro_set_controller_port_device(unsigned port, unsigned device) { (void)port; if (device == 1) ports++; }
|
||||
void retro_get_system_info(struct retro_system_info *i) {
|
||||
i->n = "Fake"; i->v = "1"; i->e = ""; i->full = true; i->block = false; }
|
||||
void retro_get_system_av_info(struct retro_system_av_info *i) {
|
||||
memset(i, 0, sizeof *i); i->t.fps = 60.0; i->t.rate = 44100.0; }
|
||||
void retro_init(void) {}
|
||||
void retro_deinit(void) {}
|
||||
bool retro_load_game(const void *g) {
|
||||
const char *dir = NULL; char path[4096]; struct retro_log_callback cb;
|
||||
const char *tag = getenv("CORE_PROBE_TEST_TAG");
|
||||
(void)g; env(9, &dir);
|
||||
snprintf(path, sizeof path, "%s/card.bin", dir);
|
||||
FILE *f = fopen(path, "rb");
|
||||
card = 0;
|
||||
if (f) { int c = fgetc(f); card = c < 0 ? 0 : (unsigned char)c; fclose(f); }
|
||||
if (env(27, &cb)) {
|
||||
logger = cb;
|
||||
cb.log(1, "card %s byte %d\n", f ? "present" : "absent", (int)card);
|
||||
cb.log(1, "tag %s\n", tag ? tag : "unset"); }
|
||||
return true; }
|
||||
void retro_unload_game(void) {}
|
||||
void retro_run(void) {
|
||||
int16_t s[4] = {1, 2, 3, 4};
|
||||
if (frames++ == 0 && logger.log) logger.log(1, "ports %u\n", ports);
|
||||
input(0, 1, 0, 8);
|
||||
for (int i = 0; i < 8; i++) pixels[i] = card;
|
||||
video(pixels, 4, 2, 8); batch(s, 2); }
|
||||
size_t retro_serialize_size(void) { return 1; }
|
||||
bool retro_serialize(void *d, size_t n) { (void)n; *(unsigned char *)d = card; return true; }
|
||||
"""
|
||||
|
||||
|
||||
def mednafen_state(sections: dict[str, dict[str, bytes]]) -> bytes:
|
||||
body = b""
|
||||
for section, entries in sections.items():
|
||||
chunk = b"".join(bytes([len(name)]) + name.encode() + struct.pack("<I", len(value)) + value
|
||||
for name, value in entries.items())
|
||||
body += section.encode().ljust(32, b"\0") + struct.pack("<I", len(chunk)) + chunk
|
||||
return b"MDFNSVST".ljust(32, b"\0") + body
|
||||
|
||||
|
||||
class ParsingTests(unittest.TestCase):
|
||||
def test_assignment(self) -> None:
|
||||
self.assertEqual(core_probe.parse_assignment("a=b=c", "--option"), ("a", "b=c"))
|
||||
self.assertEqual(core_probe.parse_assignment("dest=", "--vary"), ("dest", ""))
|
||||
for bad in ("novalue", "=x"):
|
||||
with self.assertRaises(argparse.ArgumentTypeError):
|
||||
core_probe.parse_assignment(bad, "--option")
|
||||
|
||||
def test_press(self) -> None:
|
||||
self.assertEqual(core_probe.parse_press("120:start"), (120, 3, 1))
|
||||
self.assertEqual(core_probe.parse_press("5:A:10"), (5, 8, 10))
|
||||
for bad in ("x:a", "1:turbo", "1:a:0", "-1:a", "1"):
|
||||
with self.assertRaises(argparse.ArgumentTypeError):
|
||||
core_probe.parse_press(bad)
|
||||
|
||||
def test_strace(self) -> None:
|
||||
calls = core_probe.parse_strace(STRACE_SAMPLE)
|
||||
self.assertEqual([c["path"] for c in calls],
|
||||
["/sys/bios/card.bin", "/sys/bios/missing.bin", "/sys/bios/sub/x.rom",
|
||||
"/elsewhere/game.cue"])
|
||||
self.assertEqual([c["ok"] for c in calls], [True, False, True, True])
|
||||
self.assertEqual(calls[1]["errno"], "ENOENT")
|
||||
opened, absent = core_probe.files_under(calls, Path("/sys/bios"))
|
||||
self.assertEqual(opened, ["card.bin", "sub/x.rom"])
|
||||
self.assertEqual(absent, ["missing.bin"])
|
||||
|
||||
def test_printf(self) -> None:
|
||||
name = core_probe.ctypes.create_string_buffer(b"mpeg_card.bin")
|
||||
address = core_probe.ctypes.addressof(name)
|
||||
self.assertEqual(core_probe.render_printf(b"open %s: %d%% %x\n", [address, 50, 255]),
|
||||
"open mpeg_card.bin: 50% ff")
|
||||
self.assertEqual(core_probe.render_printf(b"%d", [0xFFFFFFFF]), "-1")
|
||||
self.assertEqual(core_probe.render_printf(b"%.2f fps %u", [7]), "<double> fps 7")
|
||||
self.assertEqual(core_probe.render_printf(b"%s", [0]), "(null)")
|
||||
|
||||
def test_mednafen_state(self) -> None:
|
||||
state = b"\x00" * 7 + mednafen_state({"MPEG": {"Present": b"\x01", "AuthState": b"\x02"},
|
||||
"CDB": {"Word": b"\x34\x12"}})
|
||||
self.assertEqual(core_probe.mednafen_state_vars(state),
|
||||
{"MPEG.Present": "01", "MPEG.AuthState": "02", "CDB.Word": "3412"})
|
||||
self.assertEqual(core_probe.mednafen_state_vars(b"no state"), {})
|
||||
|
||||
def test_elf_comment(self) -> None:
|
||||
self.assertIsInstance(core_probe.elf_comment(Path(sys.executable).resolve()), list)
|
||||
with tempfile.NamedTemporaryFile(dir=REPO_ROOT / "tmp" / "tests") as handle:
|
||||
handle.write(b"not an elf")
|
||||
handle.flush()
|
||||
self.assertEqual(core_probe.elf_comment(Path(handle.name)), [])
|
||||
|
||||
def test_cue_tracks(self) -> None:
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as tmp:
|
||||
cue = Path(tmp, "d.cue")
|
||||
cue.write_text('FILE "a b.bin" BINARY\n TRACK 01 MODE2/2352\nFILE c.bin BINARY\n',
|
||||
encoding="utf-8")
|
||||
self.assertEqual([p.name for p in core_probe.cue_tracks(cue)], ["a b.bin", "c.bin"])
|
||||
|
||||
|
||||
class ResultTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def variant(label: str, value: str, flags: list[str] | None = None) -> dict:
|
||||
return {"label": label, "flags": flags or [],
|
||||
"repetitions": [{"observables": {"video_sha1": value}}]}
|
||||
|
||||
def test_same_differs_inconclusive(self) -> None:
|
||||
same = core_probe.check_result([self.variant("a", "1"), self.variant("b", "1")], "video_sha1")
|
||||
self.assertEqual(same["status"], "same")
|
||||
differs = core_probe.check_result([self.variant("a", "1"), self.variant("b", "2")], "video_sha1")
|
||||
self.assertEqual(differs["status"], "differs")
|
||||
self.assertEqual(len(differs["groups"]), 2)
|
||||
flagged = core_probe.check_result([self.variant("a", "1", ["x"]), self.variant("b", "1")],
|
||||
"video_sha1")
|
||||
self.assertEqual(flagged["status"], "inconclusive")
|
||||
|
||||
def test_timing_stats(self) -> None:
|
||||
stats = core_probe.timing_stats([1.0, 2.0, 3.0, 4.0, 10.0])
|
||||
self.assertEqual((stats["median"], stats["mean"], stats["min"], stats["max"]), (3.0, 4.0, 1.0, 10.0))
|
||||
self.assertEqual(stats["raw"], [1.0, 2.0, 3.0, 4.0, 10.0])
|
||||
|
||||
def test_refusals(self) -> None:
|
||||
parser = core_probe.build_parser()
|
||||
core = sys.executable
|
||||
bench = parser.parse_args(["bench", "--core", core, "--record", "x", "--repeat", "3"])
|
||||
self.assertIn("a recorded benchmark needs --repeat >= 5", core_probe.validate_args(bench))
|
||||
check = parser.parse_args(["check", "--core", core, "--vary", "a=", "--vary", "b=",
|
||||
"--observable", "video_sha1", "--claim", "c", "--method", "m"])
|
||||
self.assertTrue(any("same DEST" in p for p in core_probe.validate_args(check)))
|
||||
unknown = parser.parse_args(["check", "--core", core, "--vary", "a=", "--observable", "nope",
|
||||
"--claim", "c", "--method", "m"])
|
||||
self.assertTrue(any("--observable nope" in p for p in core_probe.validate_args(unknown)))
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("cc"), "no C compiler")
|
||||
class FakeCoreTests(unittest.TestCase):
|
||||
"""The whole probe against a core compiled from FAKE_CORE."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests"))
|
||||
source = cls.tmp / "fake.c"
|
||||
source.write_text(FAKE_CORE, encoding="utf-8")
|
||||
cls.core = cls.tmp / "fake_libretro.so"
|
||||
subprocess.run(["cc", "-shared", "-fPIC", "-o", str(cls.core), str(source)], check=True)
|
||||
cls.cards = []
|
||||
for byte in (b"\x01", b"\x01", b"\x02"):
|
||||
path = cls.tmp / f"card{len(cls.cards)}.bin"
|
||||
path.write_bytes(byte)
|
||||
cls.cards.append(path)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
shutil.rmtree(cls.tmp, ignore_errors=True)
|
||||
|
||||
def probe(self, argv: list[str]) -> tuple[int, dict | None]:
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
|
||||
code = core_probe.main(argv + ["--work-dir", str(self.tmp / "work")])
|
||||
return code, json.loads(out.getvalue()) if code == 0 else None
|
||||
|
||||
def test_run_record(self) -> None:
|
||||
record_dir = self.tmp / "records"
|
||||
code, record = self.probe(["check", "--core", str(self.core), "--frames", "3",
|
||||
"--vary", f"card.bin={self.cards[0]}", "--vary", f"card.bin={self.cards[1]}",
|
||||
"--vary", f"card.bin={self.cards[2]}", "--vary", "card.bin=",
|
||||
"--observable", "frame_sha1", "--claim", "c", "--method", "m",
|
||||
"--record", str(record_dir), "--max-load", "1000",
|
||||
"--env", "CORE_PROBE_TEST_TAG=set-by-test", "--press", "0:a:2"])
|
||||
self.assertEqual(code, 0)
|
||||
result = record["check"]["result"]
|
||||
self.assertEqual(result["status"], "differs")
|
||||
self.assertEqual(sorted(len(g["variants"]) for g in result["groups"]), [1, 1, 2])
|
||||
first = record["variants"][0]["repetitions"][0]
|
||||
self.assertEqual(first["logs"][0]["text"], "card present byte 1")
|
||||
self.assertEqual(first["logs"][1]["text"], "tag set-by-test")
|
||||
self.assertEqual(first["logs"][2]["text"], "ports 2")
|
||||
self.assertEqual((first["input_queries"], first["input_pressed"]), (3, 2))
|
||||
self.assertEqual(record["inputs"]["env"], {"CORE_PROBE_TEST_TAG": "set-by-test"})
|
||||
self.assertEqual(record["variants"][3]["repetitions"][0]["logs"][0]["text"], "card absent byte 0")
|
||||
if first["tracing"] == "strace":
|
||||
self.assertEqual([f["path"] for f in first["files_opened"]], ["card.bin"])
|
||||
self.assertEqual(record["variants"][3]["repetitions"][0]["files_probed_absent"], ["card.bin"])
|
||||
self.assertEqual(first["frames"], 3)
|
||||
written = list(record_dir.glob("*/*.json"))
|
||||
self.assertEqual(len(written), 1)
|
||||
self.assertEqual(written[0].parent.name, "fake")
|
||||
try:
|
||||
from jsonschema import Draft202012Validator
|
||||
except ImportError:
|
||||
return
|
||||
schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text())
|
||||
errors = list(Draft202012Validator(schema).iter_errors(json.loads(written[0].read_text())))
|
||||
self.assertEqual(errors, [])
|
||||
|
||||
def test_bench_stats(self) -> None:
|
||||
code, record = self.probe(["bench", "--core", str(self.core), "--frames", "2", "--warmup", "1",
|
||||
"--repeat", "5", "--no-trace", "--max-load", "1000"])
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(len(record["timing"]["ms_per_frame"]["raw"]), 5)
|
||||
self.assertEqual(record["variants"][0]["repetitions"][0]["tracing"], "disabled: --no-trace")
|
||||
self.assertEqual(record["flags"], [])
|
||||
|
||||
def test_busy_machine_refused(self) -> None:
|
||||
code, _ = self.probe(["run", "--core", str(self.core), "--frames", "1", "--record",
|
||||
str(self.tmp / "refused"), "--max-load", "-1"])
|
||||
self.assertEqual(code, 3)
|
||||
self.assertFalse((self.tmp / "refused").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -56,6 +56,7 @@ Every module in `tests/`, by the area it guards.
|
||||
| Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
|
||||
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_offline`, `test_install_selection`, `test_install_tui` |
|
||||
| Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` |
|
||||
| Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` |
|
||||
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
|
||||
| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
|
||||
|
||||
|
||||
@@ -704,6 +704,60 @@ python scripts/refresh_stale.py
|
||||
python scripts/refresh_stale.py --only platforms,targets --jobs 6
|
||||
```
|
||||
|
||||
### core_probe.py
|
||||
|
||||
Runs a libretro core with no window, no sound and no input device, and
|
||||
reports what it did. It is the test in real conditions behind a claim about
|
||||
a core: which files it opens, what it shows, how fast it runs.
|
||||
|
||||
```bash
|
||||
# one run, printed as JSON
|
||||
python scripts/core_probe.py run --core mednafen_saturn_libretro.so \
|
||||
--content game.cue --system-file sega_101.bin=bios/Sega/Saturn/sega_101.bin \
|
||||
--option beetle_saturn_region=Japan --frames 600
|
||||
|
||||
# timed repetitions after warm-up, written as a record
|
||||
python scripts/core_probe.py bench --core core.so --content game.cue \
|
||||
--warmup 300 --frames 3000 --repeat 5 --record measurements/
|
||||
|
||||
# one system file swapped between runs, compared on one observable
|
||||
python scripts/core_probe.py check --core core.so --content disc.cue \
|
||||
--vary mpeg_card.bin=a.bin --vary mpeg_card.bin=b.bin --vary mpeg_card.bin= \
|
||||
--observable video_sha1 --claim "..." --method "..." --record measurements/
|
||||
```
|
||||
|
||||
Each repetition runs in a fresh process under `strace -f -e trace=openat,open`
|
||||
when strace is installed. The files the core opened under the system
|
||||
directory come from the kernel, each hashed again after the run. Without
|
||||
strace the record says that tracing was unavailable.
|
||||
|
||||
| Option | Effect |
|
||||
|--------|--------|
|
||||
| `--system-file DEST=SRC` | copy SRC into a staged system directory as DEST |
|
||||
| `--option KEY=VALUE` | core option. Unset options take the first value the core declares |
|
||||
| `--frames`, `--warmup` | frames timed, and frames run before timing starts |
|
||||
| `--press FRAME:BUTTON[:LEN]` | hold a libretro joypad button on port 0 (`a`, `b`, `start`, `up`...). A menu may need 15 frames or more to see it |
|
||||
| `--joypad-ports N` | ports given a joypad after loading, as RetroArch does (default 2) |
|
||||
| `--env KEY=VALUE` | environment of the core process. A core that reads the host clock repeats only with the clock fixed, for example `LD_PRELOAD` of libfaketime with `FAKETIME` |
|
||||
| `--frame-hash-every K`, `--png-dir` | sha1 of every K-th frame, and those frames as PNG |
|
||||
| `--state-var SECTION.VAR` | a variable of a Mednafen save state, as an observable |
|
||||
| `--state-dir` | keep the save state taken at the end of the first repetition |
|
||||
| `--core-repo`, `--core-commit`, `--build-command` | where the core build comes from, copied into the record |
|
||||
| `--record DIR` | write `DIR/<core>/<date>-<sha12>.json` (schema `core-probe-record.schema.json`) |
|
||||
| `--max-load` | refuse to record when the 1-minute load average is above it (default: a quarter of the CPU count) |
|
||||
| `--no-trace` | run without strace |
|
||||
| `@FILE` | read arguments from FILE, one per line |
|
||||
|
||||
Observables are `frame_sha1` (last frame), `video_sha1` (every frame, chained),
|
||||
`audio_sha1`, `state_sha1` (save state at the end), plus `state:SECTION.VAR`
|
||||
and `frame_hashes` when asked for. A repetition that disagrees with the first
|
||||
one is flagged, and a `check` with a flagged variant reports `inconclusive`
|
||||
instead of a comparison. A record carries the core build (sha256 of the
|
||||
`.so`, the compilers its `.comment` section names), the frontend commit, the
|
||||
host (CPU, kernel, Python, load before and after) and each input by hash.
|
||||
Core log lines arrive through the libretro log interface. Floating-point
|
||||
arguments of a log line cannot be read from Python and print as `<double>`.
|
||||
|
||||
### Other tools
|
||||
|
||||
| Script | Purpose |
|
||||
|
||||
Reference in new issue
Block a user