mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 00:27:20 -05:00
chore: merge the core probe and issue 85 runs
This commit is contained in:
Abdessamad Derraz committed
commit
e627f0cf49
13 files changed
+91923
No files matched your search
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,254 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-record.schema.json",
|
||||
"title": "RetroBIOS core probe record",
|
||||
"description": "One run of scripts/core_probe.py: a libretro core executed headless, with what is needed to reproduce it and what was measured.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["record_version", "kind", "recorded_at", "core", "frontend", "host", "inputs", "variants", "flags"],
|
||||
"properties": {
|
||||
"record_version": { "const": 1 },
|
||||
"kind": { "enum": ["run", "bench", "check"] },
|
||||
"recorded_at": { "type": "string", "format": "date-time" },
|
||||
"core": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["file", "sha256", "repo", "commit", "build_command", "compiler", "library_name", "library_version"],
|
||||
"properties": {
|
||||
"file": { "type": "string", "minLength": 1 },
|
||||
"sha256": { "$ref": "#/$defs/sha256" },
|
||||
"repo": { "type": ["string", "null"], "format": "uri" },
|
||||
"commit": { "type": ["string", "null"], "pattern": "^[0-9a-f]{7,40}$" },
|
||||
"build_command": { "type": ["string", "null"] },
|
||||
"compiler": { "type": ["array", "null"], "items": { "type": "string" } },
|
||||
"library_name": { "type": ["string", "null"] },
|
||||
"library_version": { "type": ["string", "null"] }
|
||||
}
|
||||
},
|
||||
"frontend": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["script", "commit", "script_modified", "script_sha256"],
|
||||
"properties": {
|
||||
"script": { "const": "scripts/core_probe.py" },
|
||||
"commit": { "type": ["string", "null"], "pattern": "^[0-9a-f]{40}$" },
|
||||
"script_modified": { "type": ["boolean", "null"] },
|
||||
"script_sha256": { "$ref": "#/$defs/sha256" }
|
||||
}
|
||||
},
|
||||
"host": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["cpu_model", "cpu_count", "os", "kernel", "machine", "python", "load_before", "load_after", "max_load"],
|
||||
"properties": {
|
||||
"cpu_model": { "type": ["string", "null"] },
|
||||
"cpu_count": { "type": ["integer", "null"], "minimum": 1 },
|
||||
"os": { "type": "string" },
|
||||
"kernel": { "type": "string" },
|
||||
"machine": { "type": "string" },
|
||||
"python": { "type": "string" },
|
||||
"load_before": { "$ref": "#/$defs/load" },
|
||||
"load_after": { "$ref": "#/$defs/load" },
|
||||
"max_load": { "type": "number", "minimum": 0 }
|
||||
}
|
||||
},
|
||||
"inputs": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["content", "options", "env", "frames", "warmup", "repetitions", "presses", "observables", "frame_hash_every", "joypad_ports"],
|
||||
"properties": {
|
||||
"content": {
|
||||
"oneOf": [
|
||||
{ "type": "null" },
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["label", "sha1", "size", "tracks"],
|
||||
"properties": {
|
||||
"label": { "type": ["string", "null"] },
|
||||
"sha1": { "$ref": "#/$defs/sha1" },
|
||||
"size": { "type": "integer", "minimum": 0 },
|
||||
"tracks": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["file", "sha1", "size"],
|
||||
"properties": {
|
||||
"file": { "type": "string" },
|
||||
"sha1": { "$ref": "#/$defs/sha1" },
|
||||
"size": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": { "type": "object", "additionalProperties": { "type": "string" } },
|
||||
"env": { "type": "object", "additionalProperties": { "type": "string" } },
|
||||
"joypad_ports": { "type": "integer", "minimum": 0 },
|
||||
"frames": { "type": "integer", "minimum": 1 },
|
||||
"warmup": { "type": "integer", "minimum": 0 },
|
||||
"repetitions": { "type": "integer", "minimum": 1 },
|
||||
"presses": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["frame", "button", "length"],
|
||||
"properties": {
|
||||
"frame": { "type": "integer", "minimum": 0 },
|
||||
"button": { "type": "integer", "minimum": 0, "maximum": 15 },
|
||||
"length": { "type": "integer", "minimum": 1 }
|
||||
}
|
||||
}
|
||||
},
|
||||
"observables": { "type": "array", "minItems": 1, "items": { "type": "string" } },
|
||||
"frame_hash_every": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
},
|
||||
"variants": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["label", "system_files", "repetitions", "flags"],
|
||||
"properties": {
|
||||
"label": { "type": "string", "minLength": 1 },
|
||||
"system_files": {
|
||||
"type": "object",
|
||||
"additionalProperties": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/sha1" }] }
|
||||
},
|
||||
"repetitions": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/repetition" } },
|
||||
"flags": { "type": "array", "items": { "type": "string" } }
|
||||
}
|
||||
}
|
||||
},
|
||||
"flags": { "type": "array", "items": { "type": "string" } },
|
||||
"timing": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["ms_per_frame", "fps"],
|
||||
"properties": {
|
||||
"ms_per_frame": { "$ref": "#/$defs/stats" },
|
||||
"fps": { "$ref": "#/$defs/stats" }
|
||||
}
|
||||
},
|
||||
"check": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["claim", "method", "observable", "variable", "result"],
|
||||
"properties": {
|
||||
"claim": { "type": "string", "minLength": 1 },
|
||||
"method": { "type": "string", "minLength": 1 },
|
||||
"observable": { "type": "string", "minLength": 1 },
|
||||
"variable": { "type": "string", "minLength": 1 },
|
||||
"result": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["status", "statement", "groups"],
|
||||
"properties": {
|
||||
"status": { "enum": ["same", "differs", "inconclusive"] },
|
||||
"statement": { "type": "string", "minLength": 1 },
|
||||
"groups": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["value", "variants"],
|
||||
"properties": {
|
||||
"value": {},
|
||||
"variants": { "type": "array", "minItems": 1, "items": { "type": "string" } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"allOf": [
|
||||
{ "if": { "properties": { "kind": { "const": "bench" } } }, "then": { "required": ["timing"], "properties": { "inputs": { "properties": { "repetitions": { "minimum": 5 } } } } } },
|
||||
{ "if": { "properties": { "kind": { "const": "check" } } }, "then": { "required": ["check"] } }
|
||||
],
|
||||
"$defs": {
|
||||
"sha1": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
|
||||
"sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
|
||||
"load": {
|
||||
"oneOf": [
|
||||
{ "type": "null" },
|
||||
{ "type": "array", "minItems": 3, "maxItems": 3, "items": { "type": "number", "minimum": 0 } }
|
||||
]
|
||||
},
|
||||
"stats": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["median", "mean", "stdev", "min", "max", "raw"],
|
||||
"properties": {
|
||||
"median": { "type": "number" },
|
||||
"mean": { "type": "number" },
|
||||
"stdev": { "type": "number", "minimum": 0 },
|
||||
"min": { "type": "number" },
|
||||
"max": { "type": "number" },
|
||||
"raw": { "type": "array", "minItems": 1, "items": { "type": "number" } }
|
||||
}
|
||||
},
|
||||
"repetition": {
|
||||
"type": "object",
|
||||
"required": ["loaded", "library_name", "logs"],
|
||||
"properties": {
|
||||
"loaded": { "type": "boolean" },
|
||||
"library_name": { "type": "string" },
|
||||
"library_version": { "type": "string" },
|
||||
"fps_nominal": { "type": "number" },
|
||||
"frames": { "type": "integer", "minimum": 1 },
|
||||
"warmup": { "type": "integer", "minimum": 0 },
|
||||
"wall_s": { "type": "number", "minimum": 0 },
|
||||
"ms_per_frame": { "type": "number", "minimum": 0 },
|
||||
"fps": { "type": "number", "minimum": 0 },
|
||||
"observables": { "type": "object" },
|
||||
"logs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["frame", "level", "text"],
|
||||
"properties": {
|
||||
"frame": { "type": "integer", "minimum": 0 },
|
||||
"level": { "type": "string" },
|
||||
"text": { "type": "string" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"options_declared": { "type": "object" },
|
||||
"env_unanswered": { "type": "object" },
|
||||
"last_frame_size": { "type": "array", "items": { "type": "integer" } },
|
||||
"tracing": { "type": "string" },
|
||||
"input_queries": { "type": "integer", "minimum": 0 },
|
||||
"input_pressed": { "type": "integer", "minimum": 0 },
|
||||
"files_opened": {
|
||||
"oneOf": [
|
||||
{ "type": "null" },
|
||||
{
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["path", "sha1_after", "sha1_staged"],
|
||||
"properties": {
|
||||
"path": { "type": "string" },
|
||||
"sha1_after": { "$ref": "#/$defs/sha1" },
|
||||
"sha1_staged": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/sha1" }] }
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"files_probed_absent": { "oneOf": [{ "type": "null" }, { "type": "array", "items": { "type": "string" } }] }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,979 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a libretro core headless and record what it did.
|
||||
|
||||
Three modes share one runner:
|
||||
|
||||
run one execution, printed as JSON
|
||||
bench warm-up then timed repetitions, with timing statistics
|
||||
check the same execution under several variants of one system file,
|
||||
compared on a named observable
|
||||
|
||||
Each repetition runs in its own process, under strace when it is installed,
|
||||
so the files a core opened are read from the kernel rather than guessed.
|
||||
`--record DIR` writes one JSON record per invocation, carrying what is needed
|
||||
to reproduce it (core build, frontend commit, host, inputs by hash).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ctypes
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import statistics
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import zlib
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
RECORD_VERSION = 1
|
||||
MIN_BENCH_REPETITIONS = 5
|
||||
DEFAULT_WORK_DIR = ROOT / "tmp" / "core_probe"
|
||||
BASE_OBSERVABLES = ("frame_sha1", "video_sha1", "audio_sha1", "state_sha1")
|
||||
|
||||
# libretro.h environment commands the probe answers.
|
||||
ENV_GET_CAN_DUPE = 3
|
||||
ENV_GET_SYSTEM_DIRECTORY = 9
|
||||
ENV_SET_PIXEL_FORMAT = 10
|
||||
ENV_GET_VARIABLE = 15
|
||||
ENV_SET_VARIABLES = 16
|
||||
ENV_GET_VARIABLE_UPDATE = 17
|
||||
ENV_SET_SUPPORT_NO_GAME = 18
|
||||
ENV_GET_LOG_INTERFACE = 27
|
||||
ENV_GET_SAVE_DIRECTORY = 31
|
||||
ENV_GET_CORE_OPTIONS_VERSION = 52
|
||||
ENV_EXPERIMENTAL = 0x10000
|
||||
|
||||
PIXEL_BYTES = {0: 2, 1: 4, 2: 2}
|
||||
LOG_LEVELS = ("debug", "info", "warn", "error")
|
||||
JOYPAD_BUTTONS = {
|
||||
"b": 0, "y": 1, "select": 2, "start": 3, "up": 4, "down": 5, "left": 6,
|
||||
"right": 7, "a": 8, "x": 9, "l": 10, "r": 11, "l2": 12, "r2": 13,
|
||||
"l3": 14, "r3": 15,
|
||||
}
|
||||
STRACE_LINE = re.compile(
|
||||
r'^(?:\[pid\s+)?(?P<pid>\d+)\]?\s+(?P<call>openat|open)\('
|
||||
r'(?:[^,]+,\s*)?"(?P<path>(?:[^"\\]|\\.)*)"(?:\.\.\.)?,\s*(?P<flags>[^,)]+)'
|
||||
r'.*\)\s*=\s*(?P<ret>-?\d+)(?:\s+(?P<errno>[A-Z]+))?'
|
||||
)
|
||||
PRINTF_SPEC = re.compile(r"%(?P<flags>[-+ #0]*)(?P<width>\*|\d+)?(?:\.(?P<prec>\*|\d+))?"
|
||||
r"(?P<len>hh|h|ll|l|j|z|t|L)?(?P<conv>[diouxXeEfFgGaAcspn%])")
|
||||
|
||||
|
||||
class ProbeError(RuntimeError):
|
||||
"""A run that cannot produce a trustworthy result."""
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Hashing and small helpers
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def file_digest(path: Path, algorithm: str = "sha1") -> str:
|
||||
digest = hashlib.new(algorithm)
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def cue_tracks(cue: Path) -> list[Path]:
|
||||
"""Files a .cue sheet references, in order."""
|
||||
tracks: list[Path] = []
|
||||
for line in cue.read_text(encoding="utf-8", errors="replace").splitlines():
|
||||
match = re.match(r'\s*FILE\s+"([^"]+)"', line) or re.match(r"\s*FILE\s+(\S+)", line)
|
||||
if match:
|
||||
tracks.append(cue.parent / match.group(1))
|
||||
return tracks
|
||||
|
||||
|
||||
def content_identity(path: Path, label: str | None) -> dict:
|
||||
tracks = [
|
||||
{"file": track.name, "sha1": file_digest(track), "size": track.stat().st_size}
|
||||
for track in (cue_tracks(path) if path.suffix.lower() == ".cue" else [])
|
||||
]
|
||||
return {
|
||||
"label": label,
|
||||
"sha1": file_digest(path),
|
||||
"size": path.stat().st_size,
|
||||
"tracks": tracks,
|
||||
}
|
||||
|
||||
|
||||
def describe_env_value(value: str) -> str:
|
||||
"""An environment value as a record keeps it: a file by name and sha256.
|
||||
|
||||
The path is a detail of the host that ran the probe, the bytes are what
|
||||
the run depended on.
|
||||
"""
|
||||
path = Path(value)
|
||||
if path.is_absolute() and path.is_file():
|
||||
return f"{path.name} sha256:{file_digest(path, 'sha256')}"
|
||||
return value
|
||||
|
||||
|
||||
def parse_assignment(text: str, flag: str) -> tuple[str, str]:
|
||||
key, sep, value = text.partition("=")
|
||||
if not sep or not key:
|
||||
raise argparse.ArgumentTypeError(f"{flag} expects KEY=VALUE, got {text!r}")
|
||||
return key, value
|
||||
|
||||
|
||||
def parse_press(text: str) -> tuple[int, int, int]:
|
||||
"""FRAME:BUTTON[:LENGTH] on joypad port 0."""
|
||||
parts = text.split(":")
|
||||
if len(parts) not in (2, 3) or parts[1].lower() not in JOYPAD_BUTTONS:
|
||||
raise argparse.ArgumentTypeError(
|
||||
f"--press expects FRAME:BUTTON[:LENGTH] with BUTTON in "
|
||||
f"{', '.join(JOYPAD_BUTTONS)}, got {text!r}"
|
||||
)
|
||||
try:
|
||||
frame = int(parts[0])
|
||||
length = int(parts[2]) if len(parts) == 3 else 1
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError(f"--press {text!r}: {exc}") from exc
|
||||
if frame < 0 or length < 1:
|
||||
raise argparse.ArgumentTypeError(f"--press {text!r}: frame >= 0, length >= 1")
|
||||
return frame, JOYPAD_BUTTONS[parts[1].lower()], length
|
||||
|
||||
|
||||
def parse_strace(text: str) -> list[dict]:
|
||||
"""open/openat calls from `strace -f -e trace=openat,open` output."""
|
||||
calls: list[dict] = []
|
||||
for line in text.splitlines():
|
||||
match = STRACE_LINE.match(line.strip())
|
||||
if not match:
|
||||
continue
|
||||
ret = int(match.group("ret"))
|
||||
calls.append({
|
||||
"path": bytes(match.group("path"), "utf-8").decode("unicode_escape"),
|
||||
"flags": match.group("flags").strip(),
|
||||
"ok": ret >= 0,
|
||||
"errno": match.group("errno"),
|
||||
})
|
||||
return calls
|
||||
|
||||
|
||||
def files_under(calls: Iterable[dict], directory: Path) -> tuple[list[str], list[str]]:
|
||||
"""Relative paths opened, and probed but absent, under a directory."""
|
||||
base = os.path.realpath(directory)
|
||||
opened: dict[str, None] = {}
|
||||
missing: dict[str, None] = {}
|
||||
for call in calls:
|
||||
path = os.path.realpath(call["path"])
|
||||
if path != base and not path.startswith(base + os.sep):
|
||||
continue
|
||||
if os.path.isdir(path):
|
||||
continue
|
||||
rel = os.path.relpath(path, base)
|
||||
(opened if call["ok"] else missing)[rel] = None
|
||||
return list(opened), [rel for rel in missing if rel not in opened]
|
||||
|
||||
|
||||
def elf_comment(path: Path) -> list[str]:
|
||||
"""Strings of the ELF .comment section: the compilers that built the file."""
|
||||
data = path.read_bytes()
|
||||
if data[:4] != b"\x7fELF" or data[4] != 2:
|
||||
return []
|
||||
endian = "<" if data[5] == 1 else ">"
|
||||
shoff = struct.unpack_from(endian + "Q", data, 0x28)[0]
|
||||
shentsize, shnum, shstrndx = struct.unpack_from(endian + "HHH", data, 0x3A)
|
||||
sections = [struct.unpack_from(endian + "IIQQQQIIQQ", data, shoff + i * shentsize)
|
||||
for i in range(shnum)]
|
||||
if shstrndx >= len(sections):
|
||||
return []
|
||||
names_off = sections[shstrndx][4]
|
||||
for section in sections:
|
||||
name_end = data.index(b"\0", names_off + section[0])
|
||||
if data[names_off + section[0]:name_end] == b".comment":
|
||||
raw = data[section[4]:section[4] + section[5]]
|
||||
return sorted({item.decode("utf-8", "replace") for item in raw.split(b"\0") if item})
|
||||
return []
|
||||
|
||||
|
||||
def mednafen_state_vars(state: bytes) -> dict[str, str]:
|
||||
"""SECTION.VAR -> hex value from a Mednafen MDFNSVST save state."""
|
||||
start = state.find(b"MDFNSVST")
|
||||
if start < 0:
|
||||
return {}
|
||||
pos = start + 32
|
||||
out: dict[str, str] = {}
|
||||
while pos + 36 <= len(state):
|
||||
section = state[pos:pos + 32].split(b"\0", 1)[0].decode("ascii", "replace")
|
||||
size = struct.unpack_from("<I", state, pos + 32)[0]
|
||||
body = state[pos + 36:pos + 36 + size]
|
||||
pos += 36 + size
|
||||
cursor = 0
|
||||
while cursor < len(body):
|
||||
name_len = body[cursor]
|
||||
name = body[cursor + 1:cursor + 1 + name_len].decode("ascii", "replace")
|
||||
cursor += 1 + name_len
|
||||
if cursor + 4 > len(body):
|
||||
break
|
||||
value_len = struct.unpack_from("<I", body, cursor)[0] & 0x7FFFFFFF
|
||||
cursor += 4
|
||||
out[f"{section}.{name}"] = body[cursor:cursor + value_len].hex()
|
||||
cursor += value_len
|
||||
return out
|
||||
|
||||
|
||||
def render_printf(fmt: bytes, args: Sequence[int]) -> str:
|
||||
"""Render a C format string with the integer-class arguments captured.
|
||||
|
||||
Floating-point arguments travel in vector registers, which a ctypes
|
||||
callback cannot see: they are shown as <double>.
|
||||
"""
|
||||
text = fmt.decode("utf-8", "replace")
|
||||
queue = list(args)
|
||||
out: list[str] = []
|
||||
last = 0
|
||||
for match in PRINTF_SPEC.finditer(text):
|
||||
out.append(text[last:match.start()])
|
||||
last = match.end()
|
||||
conv = match.group("conv")
|
||||
if conv == "%":
|
||||
out.append("%")
|
||||
continue
|
||||
for star in (match.group("width"), match.group("prec")):
|
||||
if star == "*" and queue:
|
||||
queue.pop(0)
|
||||
if conv in "eEfFgGaA":
|
||||
out.append("<double>")
|
||||
continue
|
||||
value = queue.pop(0) if queue else 0
|
||||
if conv == "s":
|
||||
out.append(ctypes.string_at(value).decode("utf-8", "replace") if value else "(null)")
|
||||
elif conv == "c":
|
||||
out.append(chr(value & 0xFF))
|
||||
elif conv == "p":
|
||||
out.append(hex(value))
|
||||
elif conv in "di":
|
||||
bits = 64 if match.group("len") in ("l", "ll", "j", "z", "t") else 32
|
||||
value &= (1 << bits) - 1
|
||||
out.append(str(value - (1 << bits) if value >> (bits - 1) else value))
|
||||
else:
|
||||
bits = 64 if match.group("len") in ("l", "ll", "j", "z", "t") else 32
|
||||
value &= (1 << bits) - 1
|
||||
out.append({"o": f"{value:o}", "x": f"{value:x}", "X": f"{value:X}"}.get(conv, str(value)))
|
||||
out.append(text[last:])
|
||||
return "".join(out).rstrip("\n")
|
||||
|
||||
|
||||
def png_bytes(width: int, height: int, rgb: bytes) -> bytes:
|
||||
def chunk(tag: bytes, body: bytes) -> bytes:
|
||||
return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body))
|
||||
rows = b"".join(b"\0" + rgb[y * width * 3:(y + 1) * width * 3] for y in range(height))
|
||||
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(rows, 6)) + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
def to_rgb(fmt: int, width: int, height: int, packed: bytes) -> bytes:
|
||||
out = bytearray(width * height * 3)
|
||||
if fmt == 1:
|
||||
out[0::3], out[1::3], out[2::3] = packed[2::4], packed[1::4], packed[0::4]
|
||||
return bytes(out)
|
||||
for i, (pixel,) in enumerate(struct.iter_unpack("<H", packed)):
|
||||
if fmt == 2:
|
||||
r, g, b = (pixel >> 11) & 31, (pixel >> 5) & 63, pixel & 31
|
||||
out[i * 3:i * 3 + 3] = bytes((r << 3, g << 2, b << 3))
|
||||
else:
|
||||
r, g, b = (pixel >> 10) & 31, (pixel >> 5) & 31, pixel & 31
|
||||
out[i * 3:i * 3 + 3] = bytes((r << 3, g << 3, b << 3))
|
||||
return bytes(out)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# libretro bindings (worker side)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
class RetroSystemInfo(ctypes.Structure):
|
||||
_fields_ = [("library_name", ctypes.c_char_p), ("library_version", ctypes.c_char_p),
|
||||
("valid_extensions", ctypes.c_char_p), ("need_fullpath", ctypes.c_bool),
|
||||
("block_extract", ctypes.c_bool)]
|
||||
|
||||
|
||||
class RetroGameGeometry(ctypes.Structure):
|
||||
_fields_ = [("base_width", ctypes.c_uint), ("base_height", ctypes.c_uint),
|
||||
("max_width", ctypes.c_uint), ("max_height", ctypes.c_uint),
|
||||
("aspect_ratio", ctypes.c_float)]
|
||||
|
||||
|
||||
class RetroSystemTiming(ctypes.Structure):
|
||||
_fields_ = [("fps", ctypes.c_double), ("sample_rate", ctypes.c_double)]
|
||||
|
||||
|
||||
class RetroSystemAvInfo(ctypes.Structure):
|
||||
_fields_ = [("geometry", RetroGameGeometry), ("timing", RetroSystemTiming)]
|
||||
|
||||
|
||||
class RetroGameInfo(ctypes.Structure):
|
||||
_fields_ = [("path", ctypes.c_char_p), ("data", ctypes.c_void_p),
|
||||
("size", ctypes.c_size_t), ("meta", ctypes.c_char_p)]
|
||||
|
||||
|
||||
class RetroVariable(ctypes.Structure):
|
||||
_fields_ = [("key", ctypes.c_char_p), ("value", ctypes.c_char_p)]
|
||||
|
||||
|
||||
LOG_ARGS = 16
|
||||
LogFn = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p, *([ctypes.c_size_t] * LOG_ARGS))
|
||||
|
||||
|
||||
class RetroLogCallback(ctypes.Structure):
|
||||
_fields_ = [("log", LogFn)]
|
||||
|
||||
|
||||
EnvFn = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_uint, ctypes.c_void_p)
|
||||
VideoFn = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_uint, ctypes.c_uint, ctypes.c_size_t)
|
||||
AudioFn = ctypes.CFUNCTYPE(None, ctypes.c_int16, ctypes.c_int16)
|
||||
AudioBatchFn = ctypes.CFUNCTYPE(ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t)
|
||||
PollFn = ctypes.CFUNCTYPE(None)
|
||||
InputFn = ctypes.CFUNCTYPE(ctypes.c_int16, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint)
|
||||
|
||||
|
||||
class Frontend:
|
||||
"""The libretro callbacks for one run."""
|
||||
|
||||
def __init__(self, spec: dict) -> None:
|
||||
self.spec = spec
|
||||
self.system_dir = spec["system_dir"].encode()
|
||||
self.save_dir = spec["save_dir"].encode()
|
||||
self.options: dict[bytes, bytes] = {k.encode(): v.encode() for k, v in spec["options"].items()}
|
||||
self.defaults: dict[bytes, bytes] = {}
|
||||
self.declared: dict[str, list[str]] = {}
|
||||
self.option_buffers: list[ctypes.Array] = []
|
||||
self.pixel_format = 0
|
||||
self.frame = 0
|
||||
self.last_frame = b""
|
||||
self.last_size = (0, 0)
|
||||
self.video = hashlib.sha1()
|
||||
self.audio = hashlib.sha1()
|
||||
self.frame_hashes: list[list] = []
|
||||
self.logs: list[dict] = []
|
||||
self.presses: list[tuple[int, int, int]] = [tuple(p) for p in spec["presses"]]
|
||||
self.unanswered: dict[int, int] = {}
|
||||
self.input_queries = 0
|
||||
self.input_pressed = 0
|
||||
self.callbacks = (
|
||||
EnvFn(self.environment), VideoFn(self.video_refresh), AudioFn(self.audio_sample),
|
||||
AudioBatchFn(self.audio_batch), PollFn(lambda: None), InputFn(self.input_state),
|
||||
)
|
||||
self.log_callback = RetroLogCallback(LogFn(self.log))
|
||||
|
||||
def log(self, level: int, fmt: bytes, *args: int) -> None:
|
||||
name = LOG_LEVELS[level] if 0 <= level < len(LOG_LEVELS) else str(level)
|
||||
self.logs.append({"frame": self.frame, "level": name, "text": render_printf(fmt or b"", args)})
|
||||
|
||||
def environment(self, cmd: int, data: int) -> bool:
|
||||
cmd &= ~ENV_EXPERIMENTAL
|
||||
if cmd == ENV_GET_CAN_DUPE:
|
||||
ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] = True
|
||||
return True
|
||||
if cmd in (ENV_GET_SYSTEM_DIRECTORY, ENV_GET_SAVE_DIRECTORY):
|
||||
value = self.system_dir if cmd == ENV_GET_SYSTEM_DIRECTORY else self.save_dir
|
||||
buffer = ctypes.create_string_buffer(value)
|
||||
self.option_buffers.append(buffer)
|
||||
ctypes.cast(data, ctypes.POINTER(ctypes.c_char_p))[0] = ctypes.cast(buffer, ctypes.c_char_p)
|
||||
return True
|
||||
if cmd == ENV_SET_PIXEL_FORMAT:
|
||||
fmt = ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0]
|
||||
if fmt not in PIXEL_BYTES:
|
||||
return False
|
||||
self.pixel_format = fmt
|
||||
return True
|
||||
if cmd == ENV_SET_VARIABLES:
|
||||
array = ctypes.cast(data, ctypes.POINTER(RetroVariable))
|
||||
index = 0
|
||||
while array[index].key:
|
||||
key, desc = array[index].key, (array[index].value or b"").decode("utf-8", "replace")
|
||||
values = desc.split(";", 1)[-1].strip().split("|")
|
||||
self.declared[key.decode()] = values
|
||||
self.defaults[key] = values[0].encode()
|
||||
index += 1
|
||||
return True
|
||||
if cmd == ENV_GET_VARIABLE:
|
||||
var = ctypes.cast(data, ctypes.POINTER(RetroVariable))[0]
|
||||
value = self.options.get(var.key, self.defaults.get(var.key))
|
||||
if value is None:
|
||||
return False
|
||||
buffer = ctypes.create_string_buffer(value)
|
||||
self.option_buffers.append(buffer)
|
||||
ctypes.cast(data, ctypes.POINTER(RetroVariable))[0].value = ctypes.cast(buffer, ctypes.c_char_p)
|
||||
return True
|
||||
if cmd == ENV_GET_VARIABLE_UPDATE:
|
||||
ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] = False
|
||||
return True
|
||||
if cmd == ENV_SET_SUPPORT_NO_GAME:
|
||||
return True
|
||||
if cmd == ENV_GET_LOG_INTERFACE:
|
||||
ctypes.cast(data, ctypes.POINTER(RetroLogCallback))[0] = self.log_callback
|
||||
return True
|
||||
if cmd == ENV_GET_CORE_OPTIONS_VERSION:
|
||||
ctypes.cast(data, ctypes.POINTER(ctypes.c_uint))[0] = 0
|
||||
return True
|
||||
self.unanswered[cmd] = self.unanswered.get(cmd, 0) + 1
|
||||
return False
|
||||
|
||||
def video_refresh(self, data: int, width: int, height: int, pitch: int) -> None:
|
||||
if data and data != ctypes.c_void_p(-1).value:
|
||||
row = width * PIXEL_BYTES[self.pixel_format]
|
||||
raw = ctypes.string_at(data, pitch * (height - 1) + row) if height else b""
|
||||
self.last_frame = b"".join(raw[y * pitch:y * pitch + row] for y in range(height))
|
||||
self.last_size = (width, height)
|
||||
self.video.update(struct.pack("<II", *self.last_size) + hashlib.sha1(self.last_frame).digest())
|
||||
every = self.spec["frame_hash_every"]
|
||||
if every and self.frame % every == 0:
|
||||
self.frame_hashes.append([self.frame, hashlib.sha1(self.last_frame).hexdigest()])
|
||||
png_dir = self.spec.get("png_dir")
|
||||
if png_dir and self.last_frame:
|
||||
rgb = to_rgb(self.pixel_format, *self.last_size, self.last_frame)
|
||||
Path(png_dir, f"frame{self.frame:07d}.png").write_bytes(png_bytes(*self.last_size, rgb))
|
||||
|
||||
def audio_sample(self, left: int, right: int) -> None:
|
||||
self.audio.update(struct.pack("<hh", left, right))
|
||||
|
||||
def audio_batch(self, data: int, frames: int) -> int:
|
||||
self.audio.update(ctypes.string_at(data, frames * 4))
|
||||
return frames
|
||||
|
||||
def input_state(self, port: int, device: int, index: int, button: int) -> int:
|
||||
self.input_queries += 1
|
||||
if port or device != 1 or index:
|
||||
return 0
|
||||
held = int(any(start <= self.frame < start + length and button == pressed
|
||||
for start, pressed, length in self.presses))
|
||||
self.input_pressed += held
|
||||
return held
|
||||
|
||||
|
||||
def run_worker(spec: dict) -> dict:
|
||||
"""Load the core in this process and run it. Returns the measured result."""
|
||||
core = ctypes.CDLL(spec["core"])
|
||||
front = Frontend(spec)
|
||||
for name, fn in zip(("retro_set_environment", "retro_set_video_refresh", "retro_set_audio_sample",
|
||||
"retro_set_audio_sample_batch", "retro_set_input_poll", "retro_set_input_state"),
|
||||
front.callbacks):
|
||||
getattr(core, name)(fn)
|
||||
info = RetroSystemInfo()
|
||||
core.retro_get_system_info(ctypes.byref(info))
|
||||
core.retro_init()
|
||||
game = None
|
||||
if spec.get("content"):
|
||||
path = spec["content"].encode()
|
||||
game = RetroGameInfo(path, None, 0, None)
|
||||
if not info.need_fullpath:
|
||||
blob = Path(spec["content"]).read_bytes()
|
||||
holder = ctypes.create_string_buffer(blob, len(blob))
|
||||
front.option_buffers.append(holder)
|
||||
game.data, game.size = ctypes.cast(holder, ctypes.c_void_p), len(blob)
|
||||
core.retro_load_game.restype = ctypes.c_bool
|
||||
if not core.retro_load_game(ctypes.byref(game) if game else None):
|
||||
return {"loaded": False, "logs": front.logs, "library_name": info.library_name.decode()}
|
||||
# A frontend plugs a joypad into each port after loading, as RetroArch does.
|
||||
for port in range(spec["joypad_ports"]):
|
||||
core.retro_set_controller_port_device(ctypes.c_uint(port), ctypes.c_uint(1))
|
||||
av = RetroSystemAvInfo()
|
||||
core.retro_get_system_av_info(ctypes.byref(av))
|
||||
warmup, frames = spec["warmup"], spec["frames"]
|
||||
for _ in range(warmup):
|
||||
core.retro_run()
|
||||
front.frame += 1
|
||||
started = time.perf_counter()
|
||||
for _ in range(frames):
|
||||
core.retro_run()
|
||||
front.frame += 1
|
||||
wall = time.perf_counter() - started
|
||||
core.retro_serialize_size.restype = ctypes.c_size_t
|
||||
size = core.retro_serialize_size()
|
||||
state = b""
|
||||
if size:
|
||||
buffer = ctypes.create_string_buffer(size)
|
||||
core.retro_serialize.restype = ctypes.c_bool
|
||||
if core.retro_serialize(buffer, ctypes.c_size_t(size)):
|
||||
state = buffer.raw
|
||||
if spec.get("state_out"):
|
||||
Path(spec["state_out"]).write_bytes(state)
|
||||
state_vars = mednafen_state_vars(state)
|
||||
observables = {
|
||||
"frame_sha1": hashlib.sha1(front.last_frame).hexdigest(),
|
||||
"video_sha1": front.video.hexdigest(),
|
||||
"audio_sha1": front.audio.hexdigest(),
|
||||
"state_sha1": hashlib.sha1(state).hexdigest() if state else None,
|
||||
}
|
||||
for name in spec["state_vars"]:
|
||||
observables[f"state:{name}"] = state_vars.get(name)
|
||||
if spec["frame_hash_every"]:
|
||||
observables["frame_hashes"] = front.frame_hashes
|
||||
core.retro_unload_game()
|
||||
core.retro_deinit()
|
||||
return {
|
||||
"loaded": True,
|
||||
"library_name": info.library_name.decode(),
|
||||
"library_version": info.library_version.decode(),
|
||||
"fps_nominal": av.timing.fps,
|
||||
"frames": frames,
|
||||
"warmup": warmup,
|
||||
"wall_s": wall,
|
||||
"observables": observables,
|
||||
"logs": front.logs,
|
||||
"options_declared": front.declared,
|
||||
"env_unanswered": {str(k): v for k, v in sorted(front.unanswered.items())},
|
||||
"last_frame_size": list(front.last_size),
|
||||
"input_queries": front.input_queries,
|
||||
"input_pressed": front.input_pressed,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Host, core and frontend identity (parent side)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def load_average() -> list[float] | None:
|
||||
try:
|
||||
return [round(value, 2) for value in os.getloadavg()]
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def cpu_model() -> str | None:
|
||||
try:
|
||||
for line in Path("/proc/cpuinfo").read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("model name"):
|
||||
return line.split(":", 1)[1].strip()
|
||||
except OSError:
|
||||
return platform.processor() or None
|
||||
return platform.processor() or None
|
||||
|
||||
|
||||
def host_identity() -> dict:
|
||||
return {
|
||||
"cpu_model": cpu_model(),
|
||||
"cpu_count": os.cpu_count(),
|
||||
"os": platform.platform(),
|
||||
"kernel": platform.release(),
|
||||
"machine": platform.machine(),
|
||||
"python": platform.python_version(),
|
||||
}
|
||||
|
||||
|
||||
def frontend_identity() -> dict:
|
||||
script = Path(__file__).resolve()
|
||||
commit: str | None = None
|
||||
dirty: bool | None = None
|
||||
try:
|
||||
commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True,
|
||||
text=True, check=True).stdout.strip()
|
||||
status = subprocess.run(["git", "status", "--porcelain", "--", str(script)], cwd=ROOT,
|
||||
capture_output=True, text=True, check=True).stdout
|
||||
dirty = bool(status.strip())
|
||||
except (OSError, subprocess.CalledProcessError) as exc:
|
||||
print(f"WARNING: frontend commit unknown, git failed: {exc}", file=sys.stderr)
|
||||
return {"script": "scripts/core_probe.py", "commit": commit, "script_modified": dirty,
|
||||
"script_sha256": file_digest(script, "sha256")}
|
||||
|
||||
|
||||
def core_identity(args: argparse.Namespace) -> dict:
|
||||
core = Path(args.core)
|
||||
return {
|
||||
"file": core.name,
|
||||
"sha256": file_digest(core, "sha256"),
|
||||
"repo": args.core_repo,
|
||||
"commit": args.core_commit,
|
||||
"build_command": args.build_command,
|
||||
"compiler": elf_comment(core) or None,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Runs (parent side)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class Variant:
|
||||
label: str
|
||||
files: dict[str, Path | None] = field(default_factory=dict)
|
||||
|
||||
|
||||
def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> dict[str, str | None]:
|
||||
"""Copy the system files of one variant. Returns dest -> sha1 (None = absent)."""
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
staged: dict[str, str | None] = {}
|
||||
for dest, src in {**base, **variant.files}.items():
|
||||
out = target / dest
|
||||
if src is None:
|
||||
staged[dest] = None
|
||||
continue
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(src, out)
|
||||
staged[dest] = file_digest(out)
|
||||
return staged
|
||||
|
||||
|
||||
def host_paths_hidden(text: str, places: Sequence[tuple[str, str]]) -> str:
|
||||
"""A core log line with the probe's own directories named, not spelled.
|
||||
|
||||
Cores print the paths they open. Those paths belong to the host that ran
|
||||
the probe and would make two identical runs on two machines differ.
|
||||
"""
|
||||
for prefix, name in places:
|
||||
text = text.replace(prefix, name)
|
||||
return text
|
||||
|
||||
|
||||
def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None,
|
||||
png_dir: Path | None) -> dict:
|
||||
save_dir = scratch / "save"
|
||||
save_dir.mkdir(parents=True, exist_ok=True)
|
||||
spec = {
|
||||
"core": str(Path(args.core).resolve()),
|
||||
"content": str(Path(args.content).resolve()) if args.content else None,
|
||||
"system_dir": str(system_dir.resolve()),
|
||||
"save_dir": str(save_dir.resolve()),
|
||||
"options": dict(args.option),
|
||||
"frames": args.frames,
|
||||
"warmup": args.warmup,
|
||||
"presses": [list(p) for p in args.press],
|
||||
"state_vars": args.state_var,
|
||||
"frame_hash_every": args.frame_hash_every,
|
||||
"joypad_ports": args.joypad_ports,
|
||||
"png_dir": str(png_dir) if png_dir else None,
|
||||
"state_out": str(state_out) if state_out else None,
|
||||
}
|
||||
spec_path, out_path, trace_path = scratch / "spec.json", scratch / "result.json", scratch / "strace.txt"
|
||||
spec_path.write_text(json.dumps(spec), encoding="utf-8")
|
||||
command = [sys.executable, "-I", str(Path(__file__).resolve()), "_worker", str(spec_path), str(out_path)]
|
||||
strace = shutil.which("strace") if not args.no_trace else None
|
||||
if strace:
|
||||
command = [strace, "-f", "-qq", "-s", "4096", "-e", "trace=openat,open", "-o", str(trace_path)] + command
|
||||
env = {**os.environ, **dict(args.env)}
|
||||
proc = subprocess.run(command, check=False, capture_output=True, text=True,
|
||||
timeout=args.timeout, env=env)
|
||||
if proc.returncode != 0 or not out_path.exists():
|
||||
raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}")
|
||||
result = json.loads(out_path.read_text(encoding="utf-8"))
|
||||
places = [(str(system_dir.resolve()), "<system>"), (str(save_dir.resolve()), "<save>")]
|
||||
if args.content:
|
||||
places.append((str(Path(args.content).resolve().parent), "<content>"))
|
||||
places.append((str(ROOT), "<repo>"))
|
||||
for line in result.get("logs", []):
|
||||
line["text"] = host_paths_hidden(line["text"], places)
|
||||
if strace:
|
||||
opened, probed = files_under(parse_strace(trace_path.read_text(encoding="utf-8", errors="replace")),
|
||||
system_dir)
|
||||
result["files_opened"] = [{"path": rel, "sha1_after": file_digest(system_dir / rel)}
|
||||
for rel in opened if (system_dir / rel).is_file()]
|
||||
result["files_probed_absent"] = probed
|
||||
result["tracing"] = "strace"
|
||||
else:
|
||||
result["files_opened"] = None
|
||||
result["files_probed_absent"] = None
|
||||
result["tracing"] = "unavailable: strace not found" if not args.no_trace else "disabled: --no-trace"
|
||||
if result.get("wall_s") and result.get("frames"):
|
||||
result["ms_per_frame"] = result["wall_s"] * 1000 / result["frames"]
|
||||
result["fps"] = result["frames"] / result["wall_s"]
|
||||
return result
|
||||
|
||||
|
||||
def timing_stats(values: Sequence[float]) -> dict:
|
||||
return {
|
||||
"median": statistics.median(values),
|
||||
"mean": statistics.fmean(values),
|
||||
"stdev": statistics.stdev(values) if len(values) > 1 else 0.0,
|
||||
"min": min(values),
|
||||
"max": max(values),
|
||||
"raw": list(values),
|
||||
}
|
||||
|
||||
|
||||
def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Variant,
|
||||
work: Path, repetitions: int) -> dict:
|
||||
system_dir = work / "system"
|
||||
staged = stage_system_dir(base, variant, system_dir)
|
||||
reps: list[dict] = []
|
||||
flags: list[str] = []
|
||||
for index in range(repetitions):
|
||||
scratch = work / f"rep{index}"
|
||||
scratch.mkdir(parents=True, exist_ok=True)
|
||||
png_dir = None
|
||||
if args.png_dir and index == 0:
|
||||
png_dir = Path(args.png_dir) / variant.label
|
||||
png_dir.mkdir(parents=True, exist_ok=True)
|
||||
state_out = Path(args.state_dir) / f"{variant.label}.state" if args.state_dir and index == 0 else None
|
||||
if state_out:
|
||||
state_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
rep = run_once(args, system_dir, scratch, state_out, png_dir)
|
||||
if not rep.get("loaded"):
|
||||
flags.append(f"repetition {index}: retro_load_game returned false")
|
||||
for entry in rep.get("files_opened") or []:
|
||||
expected = staged.get(entry["path"])
|
||||
entry["sha1_staged"] = expected
|
||||
if expected is not None and expected != entry["sha1_after"]:
|
||||
flags.append(f"repetition {index}: {entry['path']} sha1 changed during the run")
|
||||
reps.append(rep)
|
||||
reference = reps[0].get("observables")
|
||||
for index, rep in enumerate(reps[1:], 1):
|
||||
if rep.get("observables") != reference:
|
||||
differing = sorted(k for k in (reference or {}) if (rep.get("observables") or {}).get(k) != reference[k])
|
||||
flags.append(f"repetition {index} disagrees with repetition 0 on {', '.join(differing)}")
|
||||
return {
|
||||
"label": variant.label,
|
||||
"system_files": {dest: sha for dest, sha in sorted(staged.items())},
|
||||
"repetitions": reps,
|
||||
"flags": flags,
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Command line
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
fromfile_prefix_chars="@")
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
common = argparse.ArgumentParser(add_help=False, fromfile_prefix_chars="@")
|
||||
common.add_argument("--core", required=True, help="libretro .so to load")
|
||||
common.add_argument("--content", help="content to load (a .cue is identified with its tracks)")
|
||||
common.add_argument("--content-label", help="identity of the content, e.g. its Redump name")
|
||||
common.add_argument("--system-file", action="append", default=[], metavar="DEST=SRC",
|
||||
type=lambda t: parse_assignment(t, "--system-file"),
|
||||
help="copy SRC into the staged system dir as DEST")
|
||||
common.add_argument("--option", action="append", default=[], metavar="KEY=VALUE",
|
||||
type=lambda t: parse_assignment(t, "--option"), help="core option")
|
||||
common.add_argument("--env", action="append", default=[], metavar="KEY=VALUE",
|
||||
type=lambda t: parse_assignment(t, "--env"),
|
||||
help="environment variable for the core process, e.g. LD_PRELOAD of a fixed clock")
|
||||
common.add_argument("--frames", type=int, default=600, help="frames timed after warm-up")
|
||||
common.add_argument("--warmup", type=int, default=0, help="frames run before timing starts")
|
||||
common.add_argument("--press", action="append", default=[], type=parse_press,
|
||||
metavar="FRAME:BUTTON[:LEN]", help="hold a joypad button on port 0")
|
||||
common.add_argument("--joypad-ports", type=int, default=2, metavar="N",
|
||||
help="ports given a joypad after loading (default 2)")
|
||||
common.add_argument("--state-var", action="append", default=[], metavar="SECTION.VAR",
|
||||
help="report a Mednafen save-state variable as an observable")
|
||||
common.add_argument("--frame-hash-every", type=int, default=0, metavar="K",
|
||||
help="record the sha1 of every K-th frame")
|
||||
common.add_argument("--png-dir", help="write the frames sampled by --frame-hash-every as PNG")
|
||||
common.add_argument("--state-dir", help="write the save state at the end of the first repetition")
|
||||
common.add_argument("--core-repo", help="source repository URL of the core build")
|
||||
common.add_argument("--core-commit", help="commit the core was built from")
|
||||
common.add_argument("--build-command", help="command and flags that built the core")
|
||||
common.add_argument("--record", metavar="DIR", help="write one JSON record under DIR/<core>/")
|
||||
common.add_argument("--max-load", type=float, default=None,
|
||||
help="refuse to record when the 1-minute load average exceeds this "
|
||||
"(default: a quarter of the CPU count)")
|
||||
common.add_argument("--work-dir", default=str(DEFAULT_WORK_DIR), help="scratch directory")
|
||||
common.add_argument("--timeout", type=int, default=1800, help="seconds per repetition")
|
||||
common.add_argument("--no-trace", action="store_true", help="do not run the worker under strace")
|
||||
|
||||
sub.add_parser("run", parents=[common], help="one execution")
|
||||
bench = sub.add_parser("bench", parents=[common], help="timed repetitions")
|
||||
bench.add_argument("--repeat", type=int, default=MIN_BENCH_REPETITIONS,
|
||||
help=f"timed repetitions, each in a fresh process (at least {MIN_BENCH_REPETITIONS} to record)")
|
||||
check = sub.add_parser("check", parents=[common], help="compare variants of a system file")
|
||||
check.add_argument("--vary", action="append", required=True, metavar="DEST=SRC",
|
||||
type=lambda t: parse_assignment(t, "--vary"),
|
||||
help="one variant per occurrence; an empty SRC leaves DEST absent")
|
||||
check.add_argument("--observable", required=True,
|
||||
help="the observable compared across variants")
|
||||
check.add_argument("--claim", required=True, help="the statement under test")
|
||||
check.add_argument("--method", required=True, help="how the run tests it")
|
||||
check.add_argument("--repeat", type=int, default=2, help="runs of each variant, each in a fresh process")
|
||||
worker = sub.add_parser("_worker", help="internal: one run in this process, started by the others")
|
||||
worker.add_argument("spec", help="JSON file describing the run")
|
||||
worker.add_argument("out", help="JSON file the result is written to")
|
||||
return parser
|
||||
|
||||
|
||||
def validate_args(args: argparse.Namespace) -> list[str]:
|
||||
problems: list[str] = []
|
||||
if not Path(args.core).is_file():
|
||||
problems.append(f"--core {args.core}: not a file")
|
||||
if args.content and not Path(args.content).is_file():
|
||||
problems.append(f"--content {args.content}: not a file")
|
||||
for dest, src in args.system_file:
|
||||
if not Path(src).is_file():
|
||||
problems.append(f"--system-file {dest}={src}: not a file")
|
||||
if args.frames < 1 or args.warmup < 0:
|
||||
problems.append("--frames must be >= 1 and --warmup >= 0")
|
||||
if args.png_dir and not args.frame_hash_every:
|
||||
problems.append("--png-dir needs --frame-hash-every")
|
||||
if args.mode == "bench" and args.record and args.repeat < MIN_BENCH_REPETITIONS:
|
||||
problems.append(f"a recorded benchmark needs --repeat >= {MIN_BENCH_REPETITIONS}")
|
||||
if args.mode == "check":
|
||||
dests = {dest for dest, _ in args.vary}
|
||||
if len(dests) != 1:
|
||||
problems.append("--vary must name the same DEST in every occurrence")
|
||||
for dest, src in args.vary:
|
||||
if src and not Path(src).is_file():
|
||||
problems.append(f"--vary {dest}={src}: not a file")
|
||||
names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var}
|
||||
if args.frame_hash_every:
|
||||
names.add("frame_hashes")
|
||||
if args.observable not in names:
|
||||
problems.append(f"--observable {args.observable}: not one of {', '.join(sorted(names))}")
|
||||
return problems
|
||||
|
||||
|
||||
def variants_for(args: argparse.Namespace) -> list[Variant]:
|
||||
if args.mode != "check":
|
||||
return [Variant("default")]
|
||||
out: list[Variant] = []
|
||||
seen: set[str] = set()
|
||||
for dest, src in args.vary:
|
||||
label = Path(src).name if src else f"{dest}-absent"
|
||||
while label in seen:
|
||||
label += "+"
|
||||
seen.add(label)
|
||||
out.append(Variant(label, {dest: Path(src) if src else None}))
|
||||
return out
|
||||
|
||||
|
||||
def unsettled(variant: dict, observable: str) -> bool:
|
||||
"""True when a variant cannot vouch for one value of the observable.
|
||||
|
||||
A repetition that did not load, a file changed during the run, or two
|
||||
repetitions that disagree on this observable. A disagreement on another
|
||||
observable stays in the variant's flags without voiding this one.
|
||||
"""
|
||||
reps = variant["repetitions"]
|
||||
if any(not rep.get("loaded") for rep in reps):
|
||||
return True
|
||||
if any("changed during the run" in flag for flag in variant["flags"]):
|
||||
return True
|
||||
values = {json.dumps((rep.get("observables") or {}).get(observable), sort_keys=True) for rep in reps}
|
||||
return len(values) > 1
|
||||
|
||||
|
||||
def check_result(variants: list[dict], observable: str) -> dict:
|
||||
groups: dict[str, list[str]] = {}
|
||||
failed = [v["label"] for v in variants if unsettled(v, observable)]
|
||||
for variant in variants:
|
||||
value = (variant["repetitions"][0].get("observables") or {}).get(observable)
|
||||
groups.setdefault(json.dumps(value, sort_keys=True), []).append(variant["label"])
|
||||
if failed:
|
||||
status = "inconclusive"
|
||||
statement = (f"{observable} is not settled in {len(failed)} of {len(variants)} variants "
|
||||
f"({', '.join(failed)}): no comparison is drawn")
|
||||
elif len(groups) == 1:
|
||||
status = "same"
|
||||
statement = f"{observable} is identical across all {len(variants)} variants"
|
||||
else:
|
||||
status = "differs"
|
||||
statement = f"{observable} takes {len(groups)} distinct values across {len(variants)} variants"
|
||||
other = sorted({v["label"] for v in variants if v["flags"]} - set(failed))
|
||||
if other:
|
||||
statement += f". Other flags, in {len(other)} variants, are listed with them"
|
||||
return {
|
||||
"status": status,
|
||||
"statement": statement,
|
||||
"groups": [{"value": json.loads(key), "variants": labels} for key, labels in groups.items()],
|
||||
}
|
||||
|
||||
|
||||
def record_path(directory: Path, core_name: str, record: dict) -> Path:
|
||||
canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode()
|
||||
digest = hashlib.sha256(canonical).hexdigest()[:12]
|
||||
date = record["recorded_at"][:10]
|
||||
slug = re.sub(r"[^a-z0-9]+", "_", core_name.lower()).strip("_") or "core"
|
||||
return directory / slug / f"{date}-{digest}.json"
|
||||
|
||||
|
||||
def probe(args: argparse.Namespace) -> int:
|
||||
problems = validate_args(args)
|
||||
if problems:
|
||||
for problem in problems:
|
||||
print(f"ERROR: {problem}", file=sys.stderr)
|
||||
return 2
|
||||
max_load = args.max_load if args.max_load is not None else (os.cpu_count() or 1) / 4
|
||||
load_before = load_average()
|
||||
if args.record and load_before and load_before[0] > max_load:
|
||||
print(f"ERROR: load average {load_before[0]} exceeds {max_load}: the machine is busy, "
|
||||
f"nothing recorded (raise --max-load to override)", file=sys.stderr)
|
||||
return 3
|
||||
repetitions = 1 if args.mode == "run" else args.repeat
|
||||
base = {dest: Path(src) for dest, src in args.system_file}
|
||||
work = Path(tempfile.mkdtemp(prefix=f"{args.mode}-", dir=_ensure(Path(args.work_dir))))
|
||||
try:
|
||||
variants = [run_variant(args, base, variant, work / f"v{i}", repetitions)
|
||||
for i, variant in enumerate(variants_for(args))]
|
||||
except (ProbeError, subprocess.TimeoutExpired) as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
finally:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
load_after = load_average()
|
||||
first = variants[0]["repetitions"][0]
|
||||
record: dict = {
|
||||
"record_version": RECORD_VERSION,
|
||||
"kind": args.mode,
|
||||
"recorded_at": dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat(),
|
||||
"core": {**core_identity(args), "library_name": first.get("library_name"),
|
||||
"library_version": first.get("library_version")},
|
||||
"frontend": frontend_identity(),
|
||||
"host": {**host_identity(), "load_before": load_before, "load_after": load_after,
|
||||
"max_load": max_load},
|
||||
"inputs": {
|
||||
"content": content_identity(Path(args.content), args.content_label) if args.content else None,
|
||||
"options": dict(args.option),
|
||||
"env": {key: describe_env_value(value) for key, value in args.env},
|
||||
"frames": args.frames,
|
||||
"warmup": args.warmup,
|
||||
"repetitions": repetitions,
|
||||
"presses": [{"frame": f, "button": b, "length": n} for f, b, n in args.press],
|
||||
"observables": list(BASE_OBSERVABLES) + [f"state:{v}" for v in args.state_var]
|
||||
+ (["frame_hashes"] if args.frame_hash_every else []),
|
||||
"frame_hash_every": args.frame_hash_every,
|
||||
"joypad_ports": args.joypad_ports,
|
||||
},
|
||||
"variants": variants,
|
||||
"flags": [f"{v['label']}: {flag}" for v in variants for flag in v["flags"]],
|
||||
}
|
||||
if load_before and load_after and max(load_before[0], load_after[0]) > max_load:
|
||||
record["flags"].append(f"load average reached {max(load_before[0], load_after[0])} "
|
||||
f"(threshold {max_load})")
|
||||
if args.mode == "bench":
|
||||
reps = variants[0]["repetitions"]
|
||||
record["timing"] = {"ms_per_frame": timing_stats([r["ms_per_frame"] for r in reps if "ms_per_frame" in r]),
|
||||
"fps": timing_stats([r["fps"] for r in reps if "fps" in r])}
|
||||
if args.mode == "check":
|
||||
record["check"] = {"claim": args.claim, "method": args.method, "observable": args.observable,
|
||||
"variable": args.vary[0][0],
|
||||
"result": check_result(variants, args.observable)}
|
||||
text = json.dumps(record, indent=2, sort_keys=False)
|
||||
if args.record:
|
||||
path = record_path(Path(args.record), first.get("library_name") or Path(args.core).stem, record)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text + "\n", encoding="utf-8")
|
||||
print(f"recorded {path}", file=sys.stderr)
|
||||
print(text)
|
||||
return 0
|
||||
|
||||
|
||||
def _ensure(path: Path) -> Path:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
if args.mode == "_worker":
|
||||
spec = json.loads(Path(args.spec).read_text(encoding="utf-8"))
|
||||
Path(args.out).write_text(json.dumps(run_worker(spec)), encoding="utf-8")
|
||||
return 0
|
||||
return probe(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -616,6 +616,12 @@ def main() -> int:
|
||||
errors.extend(_save_directory_blocks())
|
||||
errors.extend(_registry_contract())
|
||||
errors.extend(_event_schema_errors())
|
||||
errors.extend(
|
||||
_validate_json_files(
|
||||
sorted((ROOT / "measurements").glob("**/*.json")),
|
||||
"core-probe-record.schema.json",
|
||||
)
|
||||
)
|
||||
|
||||
if not args.source_only:
|
||||
database_path = ROOT / "database.json"
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
"""core_probe runs a libretro core headless and records what it measured.
|
||||
|
||||
The parsers are checked on fixed inputs. One test builds a ten-line
|
||||
libretro core with the C compiler, when there is one, and runs the whole
|
||||
probe on it: staging, the worker process, strace, the record and its schema.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
import core_probe
|
||||
|
||||
STRACE_SAMPLE = """\
|
||||
4242 openat(AT_FDCWD, "/sys/bios/card.bin", O_RDONLY) = 5
|
||||
4242 openat(AT_FDCWD, "/sys/bios/missing.bin", O_RDONLY) = -1 ENOENT (No such file or directory)
|
||||
[pid 4243] open("/sys/bios/sub/x.rom", O_RDONLY|O_CLOEXEC) = 7
|
||||
4242 openat(AT_FDCWD, "/elsewhere/game.cue", O_RDONLY) = 3
|
||||
4242 read(3, "...", 10) = 10
|
||||
"""
|
||||
|
||||
FAKE_CORE = r"""
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
struct retro_system_info { const char *n, *v, *e; bool full, block; };
|
||||
struct retro_game_geometry { unsigned bw, bh, mw, mh; float a; };
|
||||
struct retro_system_timing { double fps, rate; };
|
||||
struct retro_system_av_info { struct retro_game_geometry g; struct retro_system_timing t; };
|
||||
struct retro_log_callback { void (*log)(int, const char *, ...); };
|
||||
typedef bool (*env_t)(unsigned, void *);
|
||||
typedef void (*video_t)(const void *, unsigned, unsigned, size_t);
|
||||
typedef size_t (*batch_t)(const int16_t *, size_t);
|
||||
typedef int16_t (*input_t)(unsigned, unsigned, unsigned, unsigned);
|
||||
static env_t env; static video_t video; static batch_t batch; static input_t input;
|
||||
static unsigned ports, frames; static struct retro_log_callback logger;
|
||||
static uint16_t pixels[4 * 2]; static unsigned char card;
|
||||
void retro_set_environment(env_t e) { bool t = true; env = e; env(18, &t); }
|
||||
void retro_set_video_refresh(video_t v) { video = v; }
|
||||
void retro_set_audio_sample(void *f) { (void)f; }
|
||||
void retro_set_audio_sample_batch(batch_t b) { batch = b; }
|
||||
void retro_set_input_poll(void *f) { (void)f; }
|
||||
void retro_set_input_state(input_t f) { input = f; }
|
||||
void retro_set_controller_port_device(unsigned port, unsigned device) { (void)port; if (device == 1) ports++; }
|
||||
void retro_get_system_info(struct retro_system_info *i) {
|
||||
i->n = "Fake"; i->v = "1"; i->e = ""; i->full = true; i->block = false; }
|
||||
void retro_get_system_av_info(struct retro_system_av_info *i) {
|
||||
memset(i, 0, sizeof *i); i->t.fps = 60.0; i->t.rate = 44100.0; }
|
||||
void retro_init(void) {}
|
||||
void retro_deinit(void) {}
|
||||
bool retro_load_game(const void *g) {
|
||||
const char *dir = NULL; char path[4096]; struct retro_log_callback cb;
|
||||
const char *tag = getenv("CORE_PROBE_TEST_TAG");
|
||||
(void)g; env(9, &dir);
|
||||
snprintf(path, sizeof path, "%s/card.bin", dir);
|
||||
FILE *f = fopen(path, "rb");
|
||||
card = 0;
|
||||
if (f) { int c = fgetc(f); card = c < 0 ? 0 : (unsigned char)c; fclose(f); }
|
||||
if (env(27, &cb)) {
|
||||
logger = cb;
|
||||
cb.log(1, "card %s byte %d\n", f ? "present" : "absent", (int)card);
|
||||
cb.log(1, "tag %s\n", tag ? tag : "unset"); }
|
||||
return true; }
|
||||
void retro_unload_game(void) {}
|
||||
void retro_run(void) {
|
||||
int16_t s[4] = {1, 2, 3, 4};
|
||||
if (frames++ == 0 && logger.log) logger.log(1, "ports %u\n", ports);
|
||||
input(0, 1, 0, 8);
|
||||
for (int i = 0; i < 8; i++) pixels[i] = card;
|
||||
video(pixels, 4, 2, 8); batch(s, 2); }
|
||||
size_t retro_serialize_size(void) { return 1; }
|
||||
bool retro_serialize(void *d, size_t n) { (void)n; *(unsigned char *)d = card; return true; }
|
||||
"""
|
||||
|
||||
|
||||
def mednafen_state(sections: dict[str, dict[str, bytes]]) -> bytes:
|
||||
body = b""
|
||||
for section, entries in sections.items():
|
||||
chunk = b"".join(bytes([len(name)]) + name.encode() + struct.pack("<I", len(value)) + value
|
||||
for name, value in entries.items())
|
||||
body += section.encode().ljust(32, b"\0") + struct.pack("<I", len(chunk)) + chunk
|
||||
return b"MDFNSVST".ljust(32, b"\0") + body
|
||||
|
||||
|
||||
class ParsingTests(unittest.TestCase):
|
||||
def test_assignment(self) -> None:
|
||||
self.assertEqual(core_probe.parse_assignment("a=b=c", "--option"), ("a", "b=c"))
|
||||
self.assertEqual(core_probe.parse_assignment("dest=", "--vary"), ("dest", ""))
|
||||
for bad in ("novalue", "=x"):
|
||||
with self.assertRaises(argparse.ArgumentTypeError):
|
||||
core_probe.parse_assignment(bad, "--option")
|
||||
|
||||
def test_press(self) -> None:
|
||||
self.assertEqual(core_probe.parse_press("120:start"), (120, 3, 1))
|
||||
self.assertEqual(core_probe.parse_press("5:A:10"), (5, 8, 10))
|
||||
for bad in ("x:a", "1:turbo", "1:a:0", "-1:a", "1"):
|
||||
with self.assertRaises(argparse.ArgumentTypeError):
|
||||
core_probe.parse_press(bad)
|
||||
|
||||
def test_strace(self) -> None:
|
||||
calls = core_probe.parse_strace(STRACE_SAMPLE)
|
||||
self.assertEqual([c["path"] for c in calls],
|
||||
["/sys/bios/card.bin", "/sys/bios/missing.bin", "/sys/bios/sub/x.rom",
|
||||
"/elsewhere/game.cue"])
|
||||
self.assertEqual([c["ok"] for c in calls], [True, False, True, True])
|
||||
self.assertEqual(calls[1]["errno"], "ENOENT")
|
||||
opened, absent = core_probe.files_under(calls, Path("/sys/bios"))
|
||||
self.assertEqual(opened, ["card.bin", "sub/x.rom"])
|
||||
self.assertEqual(absent, ["missing.bin"])
|
||||
|
||||
def test_printf(self) -> None:
|
||||
name = core_probe.ctypes.create_string_buffer(b"mpeg_card.bin")
|
||||
address = core_probe.ctypes.addressof(name)
|
||||
self.assertEqual(core_probe.render_printf(b"open %s: %d%% %x\n", [address, 50, 255]),
|
||||
"open mpeg_card.bin: 50% ff")
|
||||
self.assertEqual(core_probe.render_printf(b"%d", [0xFFFFFFFF]), "-1")
|
||||
self.assertEqual(core_probe.render_printf(b"%.2f fps %u", [7]), "<double> fps 7")
|
||||
self.assertEqual(core_probe.render_printf(b"%s", [0]), "(null)")
|
||||
|
||||
def test_mednafen_state(self) -> None:
|
||||
state = b"\x00" * 7 + mednafen_state({"MPEG": {"Present": b"\x01", "AuthState": b"\x02"},
|
||||
"CDB": {"Word": b"\x34\x12"}})
|
||||
self.assertEqual(core_probe.mednafen_state_vars(state),
|
||||
{"MPEG.Present": "01", "MPEG.AuthState": "02", "CDB.Word": "3412"})
|
||||
self.assertEqual(core_probe.mednafen_state_vars(b"no state"), {})
|
||||
|
||||
def test_elf_comment(self) -> None:
|
||||
self.assertIsInstance(core_probe.elf_comment(Path(sys.executable).resolve()), list)
|
||||
with tempfile.NamedTemporaryFile(dir=REPO_ROOT / "tmp" / "tests") as handle:
|
||||
handle.write(b"not an elf")
|
||||
handle.flush()
|
||||
self.assertEqual(core_probe.elf_comment(Path(handle.name)), [])
|
||||
|
||||
def test_env_file_recorded_by_name_and_hash(self) -> None:
|
||||
library = Path(sys.executable).resolve()
|
||||
described = core_probe.describe_env_value(str(library))
|
||||
self.assertEqual(described, f"{library.name} sha256:{core_probe.file_digest(library, 'sha256')}")
|
||||
self.assertEqual(core_probe.describe_env_value("2000-01-01 00:00:00"), "2000-01-01 00:00:00")
|
||||
|
||||
def test_cue_tracks(self) -> None:
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as tmp:
|
||||
cue = Path(tmp, "d.cue")
|
||||
cue.write_text('FILE "a b.bin" BINARY\n TRACK 01 MODE2/2352\nFILE c.bin BINARY\n',
|
||||
encoding="utf-8")
|
||||
self.assertEqual([p.name for p in core_probe.cue_tracks(cue)], ["a b.bin", "c.bin"])
|
||||
|
||||
|
||||
class ResultTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def variant(label: str, value: str, loaded: bool = True) -> dict:
|
||||
return {"label": label, "flags": [],
|
||||
"repetitions": [{"loaded": loaded, "observables": {"video_sha1": value}}]}
|
||||
|
||||
def test_same_differs_inconclusive(self) -> None:
|
||||
same = core_probe.check_result([self.variant("a", "1"), self.variant("b", "1")], "video_sha1")
|
||||
self.assertEqual(same["status"], "same")
|
||||
differs = core_probe.check_result([self.variant("a", "1"), self.variant("b", "2")], "video_sha1")
|
||||
self.assertEqual(differs["status"], "differs")
|
||||
self.assertEqual(len(differs["groups"]), 2)
|
||||
flagged = core_probe.check_result([self.variant("a", "1", loaded=False), self.variant("b", "1")],
|
||||
"video_sha1")
|
||||
self.assertEqual(flagged["status"], "inconclusive")
|
||||
|
||||
def test_other_observable_does_not_void_the_check(self) -> None:
|
||||
variant = {"label": "a", "flags": ["repetition 1 disagrees with repetition 0 on state_sha1"],
|
||||
"repetitions": [{"loaded": True, "observables": {"video_sha1": "1", "state_sha1": "x"}},
|
||||
{"loaded": True, "observables": {"video_sha1": "1", "state_sha1": "y"}}]}
|
||||
result = core_probe.check_result([variant], "video_sha1")
|
||||
self.assertEqual(result["status"], "same")
|
||||
self.assertIn("Other flags", result["statement"])
|
||||
self.assertEqual(core_probe.check_result([variant], "state_sha1")["status"], "inconclusive")
|
||||
|
||||
def test_host_paths_hidden(self) -> None:
|
||||
places = [("/tmp/x/system", "<system>"), ("/tmp/x", "<repo>")]
|
||||
self.assertEqual(core_probe.host_paths_hidden("/tmp/x/system/bios.bin NOT FOUND", places),
|
||||
"<system>/bios.bin NOT FOUND")
|
||||
|
||||
def test_timing_stats(self) -> None:
|
||||
stats = core_probe.timing_stats([1.0, 2.0, 3.0, 4.0, 10.0])
|
||||
self.assertEqual((stats["median"], stats["mean"], stats["min"], stats["max"]), (3.0, 4.0, 1.0, 10.0))
|
||||
self.assertEqual(stats["raw"], [1.0, 2.0, 3.0, 4.0, 10.0])
|
||||
|
||||
def test_refusals(self) -> None:
|
||||
parser = core_probe.build_parser()
|
||||
core = sys.executable
|
||||
bench = parser.parse_args(["bench", "--core", core, "--record", "x", "--repeat", "3"])
|
||||
self.assertIn("a recorded benchmark needs --repeat >= 5", core_probe.validate_args(bench))
|
||||
check = parser.parse_args(["check", "--core", core, "--vary", "a=", "--vary", "b=",
|
||||
"--observable", "video_sha1", "--claim", "c", "--method", "m"])
|
||||
self.assertTrue(any("same DEST" in p for p in core_probe.validate_args(check)))
|
||||
unknown = parser.parse_args(["check", "--core", core, "--vary", "a=", "--observable", "nope",
|
||||
"--claim", "c", "--method", "m"])
|
||||
self.assertTrue(any("--observable nope" in p for p in core_probe.validate_args(unknown)))
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("cc"), "no C compiler")
|
||||
class FakeCoreTests(unittest.TestCase):
|
||||
"""The whole probe against a core compiled from FAKE_CORE."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests"))
|
||||
source = cls.tmp / "fake.c"
|
||||
source.write_text(FAKE_CORE, encoding="utf-8")
|
||||
cls.core = cls.tmp / "fake_libretro.so"
|
||||
subprocess.run(["cc", "-shared", "-fPIC", "-o", str(cls.core), str(source)], check=True)
|
||||
cls.cards = []
|
||||
for byte in (b"\x01", b"\x01", b"\x02"):
|
||||
path = cls.tmp / f"card{len(cls.cards)}.bin"
|
||||
path.write_bytes(byte)
|
||||
cls.cards.append(path)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
shutil.rmtree(cls.tmp, ignore_errors=True)
|
||||
|
||||
def probe(self, argv: list[str]) -> tuple[int, dict | None]:
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
|
||||
code = core_probe.main(argv + ["--work-dir", str(self.tmp / "work")])
|
||||
return code, json.loads(out.getvalue()) if code == 0 else None
|
||||
|
||||
def test_run_record(self) -> None:
|
||||
record_dir = self.tmp / "records"
|
||||
code, record = self.probe(["check", "--core", str(self.core), "--frames", "3",
|
||||
"--vary", f"card.bin={self.cards[0]}", "--vary", f"card.bin={self.cards[1]}",
|
||||
"--vary", f"card.bin={self.cards[2]}", "--vary", "card.bin=",
|
||||
"--observable", "frame_sha1", "--claim", "c", "--method", "m",
|
||||
"--record", str(record_dir), "--max-load", "1000",
|
||||
"--env", "CORE_PROBE_TEST_TAG=set-by-test", "--press", "0:a:2"])
|
||||
self.assertEqual(code, 0)
|
||||
result = record["check"]["result"]
|
||||
self.assertEqual(result["status"], "differs")
|
||||
self.assertEqual(sorted(len(g["variants"]) for g in result["groups"]), [1, 1, 2])
|
||||
first = record["variants"][0]["repetitions"][0]
|
||||
self.assertEqual(first["logs"][0]["text"], "card present byte 1")
|
||||
self.assertEqual(first["logs"][1]["text"], "tag set-by-test")
|
||||
self.assertEqual(first["logs"][2]["text"], "ports 2")
|
||||
self.assertEqual((first["input_queries"], first["input_pressed"]), (3, 2))
|
||||
self.assertEqual(record["inputs"]["env"], {"CORE_PROBE_TEST_TAG": "set-by-test"})
|
||||
self.assertEqual(record["variants"][3]["repetitions"][0]["logs"][0]["text"], "card absent byte 0")
|
||||
if first["tracing"] == "strace":
|
||||
self.assertEqual([f["path"] for f in first["files_opened"]], ["card.bin"])
|
||||
self.assertEqual(record["variants"][3]["repetitions"][0]["files_probed_absent"], ["card.bin"])
|
||||
self.assertEqual(first["frames"], 3)
|
||||
written = list(record_dir.glob("*/*.json"))
|
||||
self.assertEqual(len(written), 1)
|
||||
self.assertEqual(written[0].parent.name, "fake")
|
||||
try:
|
||||
from jsonschema import Draft202012Validator
|
||||
except ImportError:
|
||||
return
|
||||
schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text())
|
||||
errors = list(Draft202012Validator(schema).iter_errors(json.loads(written[0].read_text())))
|
||||
self.assertEqual(errors, [])
|
||||
|
||||
def test_bench_stats(self) -> None:
|
||||
code, record = self.probe(["bench", "--core", str(self.core), "--frames", "2", "--warmup", "1",
|
||||
"--repeat", "5", "--no-trace", "--max-load", "1000"])
|
||||
self.assertEqual(code, 0)
|
||||
self.assertEqual(len(record["timing"]["ms_per_frame"]["raw"]), 5)
|
||||
self.assertEqual(record["variants"][0]["repetitions"][0]["tracing"], "disabled: --no-trace")
|
||||
self.assertEqual(record["flags"], [])
|
||||
|
||||
def test_busy_machine_refused(self) -> None:
|
||||
code, _ = self.probe(["run", "--core", str(self.core), "--frames", "1", "--record",
|
||||
str(self.tmp / "refused"), "--max-load", "-1"])
|
||||
self.assertEqual(code, 3)
|
||||
self.assertFalse((self.tmp / "refused").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -56,6 +56,7 @@ Every module in `tests/`, by the area it guards.
|
||||
| Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` |
|
||||
| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_offline`, `test_install_selection`, `test_install_tui` |
|
||||
| Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` |
|
||||
| Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` |
|
||||
| Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` |
|
||||
| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_search`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` |
|
||||
|
||||
|
||||
@@ -704,6 +704,63 @@ python scripts/refresh_stale.py
|
||||
python scripts/refresh_stale.py --only platforms,targets --jobs 6
|
||||
```
|
||||
|
||||
### core_probe.py
|
||||
|
||||
Runs a libretro core with no window, no sound and no input device, and
|
||||
reports what it did. It is the test in real conditions behind a claim about
|
||||
a core: which files it opens, what it shows, how fast it runs.
|
||||
|
||||
```bash
|
||||
# one run, printed as JSON
|
||||
python scripts/core_probe.py run --core mednafen_saturn_libretro.so \
|
||||
--content game.cue --system-file sega_101.bin=bios/Sega/Saturn/sega_101.bin \
|
||||
--option beetle_saturn_region=Japan --frames 600
|
||||
|
||||
# timed repetitions after warm-up, written as a record
|
||||
python scripts/core_probe.py bench --core core.so --content game.cue \
|
||||
--warmup 300 --frames 3000 --repeat 5 --record measurements/
|
||||
|
||||
# one system file swapped between runs, compared on one observable
|
||||
python scripts/core_probe.py check --core core.so --content disc.cue \
|
||||
--vary mpeg_card.bin=a.bin --vary mpeg_card.bin=b.bin --vary mpeg_card.bin= \
|
||||
--observable video_sha1 --claim "..." --method "..." --record measurements/
|
||||
```
|
||||
|
||||
Each repetition runs in a fresh process under `strace -f -e trace=openat,open`
|
||||
when strace is installed. The files the core opened under the system
|
||||
directory come from the kernel, each hashed again after the run. Without
|
||||
strace the record says that tracing was unavailable.
|
||||
|
||||
| Option | Effect |
|
||||
|--------|--------|
|
||||
| `--system-file DEST=SRC` | copy SRC into a staged system directory as DEST |
|
||||
| `--option KEY=VALUE` | core option. Unset options take the first value the core declares |
|
||||
| `--frames`, `--warmup` | frames timed, and frames run before timing starts |
|
||||
| `--press FRAME:BUTTON[:LEN]` | hold a libretro joypad button on port 0 (`a`, `b`, `start`, `up`...). A menu may need 15 frames or more to see it |
|
||||
| `--joypad-ports N` | ports given a joypad after loading, as RetroArch does (default 2) |
|
||||
| `--env KEY=VALUE` | environment of the core process. A core that reads the host clock repeats only with the clock fixed, for example `LD_PRELOAD` of libfaketime with `FAKETIME`. A value naming a file is recorded by file name and sha256 |
|
||||
| `--frame-hash-every K`, `--png-dir` | sha1 of every K-th frame, and those frames as PNG |
|
||||
| `--state-var SECTION.VAR` | a variable of a Mednafen save state, as an observable |
|
||||
| `--state-dir` | keep the save state taken at the end of the first repetition |
|
||||
| `--core-repo`, `--core-commit`, `--build-command` | where the core build comes from, copied into the record |
|
||||
| `--record DIR` | write `DIR/<core>/<date>-<sha12>.json` (schema `core-probe-record.schema.json`) |
|
||||
| `--max-load` | refuse to record when the 1-minute load average is above it (default: a quarter of the CPU count) |
|
||||
| `--no-trace` | run without strace |
|
||||
| `@FILE` | read arguments from FILE, one per line |
|
||||
|
||||
Observables are `frame_sha1` (last frame), `video_sha1` (every frame, chained),
|
||||
`audio_sha1`, `state_sha1` (save state at the end), plus `state:SECTION.VAR`
|
||||
and `frame_hashes` when asked for. A repetition that disagrees with the first
|
||||
one is flagged. A `check` reports `inconclusive` instead of a comparison when,
|
||||
in any variant, the compared observable differs between repetitions, a run did
|
||||
not load, or a file changed during the run. A disagreement on another
|
||||
observable stays flagged in the record. In core log lines, the staged system,
|
||||
save and content directories read `<system>`, `<save>` and `<content>`. A record carries the core build (sha256 of the
|
||||
`.so`, the compilers its `.comment` section names), the frontend commit, the
|
||||
host (CPU, kernel, Python, load before and after) and each input by hash.
|
||||
Core log lines arrive through the libretro log interface. Floating-point
|
||||
arguments of a log line cannot be read from Python and print as `<double>`.
|
||||
|
||||
### Other tools
|
||||
|
||||
| Script | Purpose |
|
||||
|
||||
Reference in new issue
Block a user