From 00b034612f33617ff742e77cbc60df5be2b942af Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 05:18:10 +0100 Subject: [PATCH] feat: add headless libretro core probe core_probe.py loads a libretro core with no window, runs it for a number of frames and reports the files it opened (strace), its log lines, frame, audio and save-state hashes and its speed. bench repeats a run after warm-up and gives median, mean, deviation and range; check swaps one system file between runs and compares one observable. --record writes a JSON record with the core build, frontend commit, host and every input by hash, validated against core-probe-record.schema.json. A busy host is refused, and a repetition that disagrees with the first is flagged. --- schemas/core-probe-record.schema.json | 254 +++++++ scripts/core_probe.py | 930 ++++++++++++++++++++++++++ scripts/validate_schemas.py | 6 + tests/test_core_probe.py | 269 ++++++++ wiki/testing-guide.md | 1 + wiki/tools.md | 54 ++ 6 files changed, 1514 insertions(+) create mode 100644 schemas/core-probe-record.schema.json create mode 100644 scripts/core_probe.py create mode 100644 tests/test_core_probe.py diff --git a/schemas/core-probe-record.schema.json b/schemas/core-probe-record.schema.json new file mode 100644 index 000000000..22e4c9529 --- /dev/null +++ b/schemas/core-probe-record.schema.json @@ -0,0 +1,254 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/core-probe-record.schema.json", + "title": "RetroBIOS core probe record", + "description": "One run of scripts/core_probe.py: a libretro core executed headless, with what is needed to reproduce it and what was measured.", + "type": "object", + "additionalProperties": false, + "required": ["record_version", "kind", "recorded_at", "core", "frontend", "host", "inputs", "variants", "flags"], + "properties": { + "record_version": { "const": 1 }, + "kind": { "enum": ["run", "bench", "check"] }, + "recorded_at": { "type": "string", "format": "date-time" }, + "core": { + "type": "object", + "additionalProperties": false, + "required": ["file", "sha256", "repo", "commit", "build_command", "compiler", "library_name", "library_version"], + "properties": { + "file": { "type": "string", "minLength": 1 }, + "sha256": { "$ref": "#/$defs/sha256" }, + "repo": { "type": ["string", "null"], "format": "uri" }, + "commit": { "type": ["string", "null"], "pattern": "^[0-9a-f]{7,40}$" }, + "build_command": { "type": ["string", "null"] }, + "compiler": { "type": ["array", "null"], "items": { "type": "string" } }, + "library_name": { "type": ["string", "null"] }, + "library_version": { "type": ["string", "null"] } + } + }, + "frontend": { + "type": "object", + "additionalProperties": false, + "required": ["script", "commit", "script_modified", "script_sha256"], + "properties": { + "script": { "const": "scripts/core_probe.py" }, + "commit": { "type": ["string", "null"], "pattern": "^[0-9a-f]{40}$" }, + "script_modified": { "type": ["boolean", "null"] }, + "script_sha256": { "$ref": "#/$defs/sha256" } + } + }, + "host": { + "type": "object", + "additionalProperties": false, + "required": ["cpu_model", "cpu_count", "os", "kernel", "machine", "python", "load_before", "load_after", "max_load"], + "properties": { + "cpu_model": { "type": ["string", "null"] }, + "cpu_count": { "type": ["integer", "null"], "minimum": 1 }, + "os": { "type": "string" }, + "kernel": { "type": "string" }, + "machine": { "type": "string" }, + "python": { "type": "string" }, + "load_before": { "$ref": "#/$defs/load" }, + "load_after": { "$ref": "#/$defs/load" }, + "max_load": { "type": "number", "minimum": 0 } + } + }, + "inputs": { + "type": "object", + "additionalProperties": false, + "required": ["content", "options", "env", "frames", "warmup", "repetitions", "presses", "observables", "frame_hash_every", "joypad_ports"], + "properties": { + "content": { + "oneOf": [ + { "type": "null" }, + { + "type": "object", + "additionalProperties": false, + "required": ["label", "sha1", "size", "tracks"], + "properties": { + "label": { "type": ["string", "null"] }, + "sha1": { "$ref": "#/$defs/sha1" }, + "size": { "type": "integer", "minimum": 0 }, + "tracks": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["file", "sha1", "size"], + "properties": { + "file": { "type": "string" }, + "sha1": { "$ref": "#/$defs/sha1" }, + "size": { "type": "integer", "minimum": 0 } + } + } + } + } + } + ] + }, + "options": { "type": "object", "additionalProperties": { "type": "string" } }, + "env": { "type": "object", "additionalProperties": { "type": "string" } }, + "joypad_ports": { "type": "integer", "minimum": 0 }, + "frames": { "type": "integer", "minimum": 1 }, + "warmup": { "type": "integer", "minimum": 0 }, + "repetitions": { "type": "integer", "minimum": 1 }, + "presses": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["frame", "button", "length"], + "properties": { + "frame": { "type": "integer", "minimum": 0 }, + "button": { "type": "integer", "minimum": 0, "maximum": 15 }, + "length": { "type": "integer", "minimum": 1 } + } + } + }, + "observables": { "type": "array", "minItems": 1, "items": { "type": "string" } }, + "frame_hash_every": { "type": "integer", "minimum": 0 } + } + }, + "variants": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["label", "system_files", "repetitions", "flags"], + "properties": { + "label": { "type": "string", "minLength": 1 }, + "system_files": { + "type": "object", + "additionalProperties": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/sha1" }] } + }, + "repetitions": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/repetition" } }, + "flags": { "type": "array", "items": { "type": "string" } } + } + } + }, + "flags": { "type": "array", "items": { "type": "string" } }, + "timing": { + "type": "object", + "additionalProperties": false, + "required": ["ms_per_frame", "fps"], + "properties": { + "ms_per_frame": { "$ref": "#/$defs/stats" }, + "fps": { "$ref": "#/$defs/stats" } + } + }, + "check": { + "type": "object", + "additionalProperties": false, + "required": ["claim", "method", "observable", "variable", "result"], + "properties": { + "claim": { "type": "string", "minLength": 1 }, + "method": { "type": "string", "minLength": 1 }, + "observable": { "type": "string", "minLength": 1 }, + "variable": { "type": "string", "minLength": 1 }, + "result": { + "type": "object", + "additionalProperties": false, + "required": ["status", "statement", "groups"], + "properties": { + "status": { "enum": ["same", "differs", "inconclusive"] }, + "statement": { "type": "string", "minLength": 1 }, + "groups": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["value", "variants"], + "properties": { + "value": {}, + "variants": { "type": "array", "minItems": 1, "items": { "type": "string" } } + } + } + } + } + } + } + } + }, + "allOf": [ + { "if": { "properties": { "kind": { "const": "bench" } } }, "then": { "required": ["timing"], "properties": { "inputs": { "properties": { "repetitions": { "minimum": 5 } } } } } }, + { "if": { "properties": { "kind": { "const": "check" } } }, "then": { "required": ["check"] } } + ], + "$defs": { + "sha1": { "type": "string", "pattern": "^[0-9a-f]{40}$" }, + "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "load": { + "oneOf": [ + { "type": "null" }, + { "type": "array", "minItems": 3, "maxItems": 3, "items": { "type": "number", "minimum": 0 } } + ] + }, + "stats": { + "type": "object", + "additionalProperties": false, + "required": ["median", "mean", "stdev", "min", "max", "raw"], + "properties": { + "median": { "type": "number" }, + "mean": { "type": "number" }, + "stdev": { "type": "number", "minimum": 0 }, + "min": { "type": "number" }, + "max": { "type": "number" }, + "raw": { "type": "array", "minItems": 1, "items": { "type": "number" } } + } + }, + "repetition": { + "type": "object", + "required": ["loaded", "library_name", "logs"], + "properties": { + "loaded": { "type": "boolean" }, + "library_name": { "type": "string" }, + "library_version": { "type": "string" }, + "fps_nominal": { "type": "number" }, + "frames": { "type": "integer", "minimum": 1 }, + "warmup": { "type": "integer", "minimum": 0 }, + "wall_s": { "type": "number", "minimum": 0 }, + "ms_per_frame": { "type": "number", "minimum": 0 }, + "fps": { "type": "number", "minimum": 0 }, + "observables": { "type": "object" }, + "logs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["frame", "level", "text"], + "properties": { + "frame": { "type": "integer", "minimum": 0 }, + "level": { "type": "string" }, + "text": { "type": "string" } + } + } + }, + "options_declared": { "type": "object" }, + "env_unanswered": { "type": "object" }, + "last_frame_size": { "type": "array", "items": { "type": "integer" } }, + "tracing": { "type": "string" }, + "input_queries": { "type": "integer", "minimum": 0 }, + "input_pressed": { "type": "integer", "minimum": 0 }, + "files_opened": { + "oneOf": [ + { "type": "null" }, + { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["path", "sha1_after", "sha1_staged"], + "properties": { + "path": { "type": "string" }, + "sha1_after": { "$ref": "#/$defs/sha1" }, + "sha1_staged": { "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/sha1" }] } + } + } + } + ] + }, + "files_probed_absent": { "oneOf": [{ "type": "null" }, { "type": "array", "items": { "type": "string" } }] } + } + } + } +} diff --git a/scripts/core_probe.py b/scripts/core_probe.py new file mode 100644 index 000000000..e2b303899 --- /dev/null +++ b/scripts/core_probe.py @@ -0,0 +1,930 @@ +#!/usr/bin/env python3 +"""Run a libretro core headless and record what it did. + +Three modes share one runner: + + run one execution, printed as JSON + bench warm-up then timed repetitions, with timing statistics + check the same execution under several variants of one system file, + compared on a named observable + +Each repetition runs in its own process, under strace when it is installed, +so the files a core opened are read from the kernel rather than guessed. +`--record DIR` writes one JSON record per invocation, carrying what is needed +to reproduce it (core build, frontend commit, host, inputs by hash). +""" + +from __future__ import annotations + +import argparse +import ctypes +import datetime as dt +import hashlib +import json +import os +import platform +import re +import shutil +import statistics +import struct +import subprocess +import sys +import tempfile +import time +import zlib +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RECORD_VERSION = 1 +MIN_BENCH_REPETITIONS = 5 +DEFAULT_WORK_DIR = ROOT / "tmp" / "core_probe" +BASE_OBSERVABLES = ("frame_sha1", "video_sha1", "audio_sha1", "state_sha1") + +# libretro.h environment commands the probe answers. +ENV_GET_CAN_DUPE = 3 +ENV_GET_SYSTEM_DIRECTORY = 9 +ENV_SET_PIXEL_FORMAT = 10 +ENV_GET_VARIABLE = 15 +ENV_SET_VARIABLES = 16 +ENV_GET_VARIABLE_UPDATE = 17 +ENV_SET_SUPPORT_NO_GAME = 18 +ENV_GET_LOG_INTERFACE = 27 +ENV_GET_SAVE_DIRECTORY = 31 +ENV_GET_CORE_OPTIONS_VERSION = 52 +ENV_EXPERIMENTAL = 0x10000 + +PIXEL_BYTES = {0: 2, 1: 4, 2: 2} +LOG_LEVELS = ("debug", "info", "warn", "error") +JOYPAD_BUTTONS = { + "b": 0, "y": 1, "select": 2, "start": 3, "up": 4, "down": 5, "left": 6, + "right": 7, "a": 8, "x": 9, "l": 10, "r": 11, "l2": 12, "r2": 13, + "l3": 14, "r3": 15, +} +STRACE_LINE = re.compile( + r'^(?:\[pid\s+)?(?P\d+)\]?\s+(?Popenat|open)\(' + r'(?:[^,]+,\s*)?"(?P(?:[^"\\]|\\.)*)"(?:\.\.\.)?,\s*(?P[^,)]+)' + r'.*\)\s*=\s*(?P-?\d+)(?:\s+(?P[A-Z]+))?' +) +PRINTF_SPEC = re.compile(r"%(?P[-+ #0]*)(?P\*|\d+)?(?:\.(?P\*|\d+))?" + r"(?Phh|h|ll|l|j|z|t|L)?(?P[diouxXeEfFgGaAcspn%])") + + +class ProbeError(RuntimeError): + """A run that cannot produce a trustworthy result.""" + + +# -------------------------------------------------------------------------- +# Hashing and small helpers +# -------------------------------------------------------------------------- + +def file_digest(path: Path, algorithm: str = "sha1") -> str: + digest = hashlib.new(algorithm) + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def cue_tracks(cue: Path) -> list[Path]: + """Files a .cue sheet references, in order.""" + tracks: list[Path] = [] + for line in cue.read_text(encoding="utf-8", errors="replace").splitlines(): + match = re.match(r'\s*FILE\s+"([^"]+)"', line) or re.match(r"\s*FILE\s+(\S+)", line) + if match: + tracks.append(cue.parent / match.group(1)) + return tracks + + +def content_identity(path: Path, label: str | None) -> dict: + tracks = [ + {"file": track.name, "sha1": file_digest(track), "size": track.stat().st_size} + for track in (cue_tracks(path) if path.suffix.lower() == ".cue" else []) + ] + return { + "label": label, + "sha1": file_digest(path), + "size": path.stat().st_size, + "tracks": tracks, + } + + +def parse_assignment(text: str, flag: str) -> tuple[str, str]: + key, sep, value = text.partition("=") + if not sep or not key: + raise argparse.ArgumentTypeError(f"{flag} expects KEY=VALUE, got {text!r}") + return key, value + + +def parse_press(text: str) -> tuple[int, int, int]: + """FRAME:BUTTON[:LENGTH] on joypad port 0.""" + parts = text.split(":") + if len(parts) not in (2, 3) or parts[1].lower() not in JOYPAD_BUTTONS: + raise argparse.ArgumentTypeError( + f"--press expects FRAME:BUTTON[:LENGTH] with BUTTON in " + f"{', '.join(JOYPAD_BUTTONS)}, got {text!r}" + ) + try: + frame = int(parts[0]) + length = int(parts[2]) if len(parts) == 3 else 1 + except ValueError as exc: + raise argparse.ArgumentTypeError(f"--press {text!r}: {exc}") from exc + if frame < 0 or length < 1: + raise argparse.ArgumentTypeError(f"--press {text!r}: frame >= 0, length >= 1") + return frame, JOYPAD_BUTTONS[parts[1].lower()], length + + +def parse_strace(text: str) -> list[dict]: + """open/openat calls from `strace -f -e trace=openat,open` output.""" + calls: list[dict] = [] + for line in text.splitlines(): + match = STRACE_LINE.match(line.strip()) + if not match: + continue + ret = int(match.group("ret")) + calls.append({ + "path": bytes(match.group("path"), "utf-8").decode("unicode_escape"), + "flags": match.group("flags").strip(), + "ok": ret >= 0, + "errno": match.group("errno"), + }) + return calls + + +def files_under(calls: Iterable[dict], directory: Path) -> tuple[list[str], list[str]]: + """Relative paths opened, and probed but absent, under a directory.""" + base = os.path.realpath(directory) + opened: dict[str, None] = {} + missing: dict[str, None] = {} + for call in calls: + path = os.path.realpath(call["path"]) + if path != base and not path.startswith(base + os.sep): + continue + if os.path.isdir(path): + continue + rel = os.path.relpath(path, base) + (opened if call["ok"] else missing)[rel] = None + return list(opened), [rel for rel in missing if rel not in opened] + + +def elf_comment(path: Path) -> list[str]: + """Strings of the ELF .comment section: the compilers that built the file.""" + data = path.read_bytes() + if data[:4] != b"\x7fELF" or data[4] != 2: + return [] + endian = "<" if data[5] == 1 else ">" + shoff = struct.unpack_from(endian + "Q", data, 0x28)[0] + shentsize, shnum, shstrndx = struct.unpack_from(endian + "HHH", data, 0x3A) + sections = [struct.unpack_from(endian + "IIQQQQIIQQ", data, shoff + i * shentsize) + for i in range(shnum)] + if shstrndx >= len(sections): + return [] + names_off = sections[shstrndx][4] + for section in sections: + name_end = data.index(b"\0", names_off + section[0]) + if data[names_off + section[0]:name_end] == b".comment": + raw = data[section[4]:section[4] + section[5]] + return sorted({item.decode("utf-8", "replace") for item in raw.split(b"\0") if item}) + return [] + + +def mednafen_state_vars(state: bytes) -> dict[str, str]: + """SECTION.VAR -> hex value from a Mednafen MDFNSVST save state.""" + start = state.find(b"MDFNSVST") + if start < 0: + return {} + pos = start + 32 + out: dict[str, str] = {} + while pos + 36 <= len(state): + section = state[pos:pos + 32].split(b"\0", 1)[0].decode("ascii", "replace") + size = struct.unpack_from(" len(body): + break + value_len = struct.unpack_from(" str: + """Render a C format string with the integer-class arguments captured. + + Floating-point arguments travel in vector registers, which a ctypes + callback cannot see: they are shown as . + """ + text = fmt.decode("utf-8", "replace") + queue = list(args) + out: list[str] = [] + last = 0 + for match in PRINTF_SPEC.finditer(text): + out.append(text[last:match.start()]) + last = match.end() + conv = match.group("conv") + if conv == "%": + out.append("%") + continue + for star in (match.group("width"), match.group("prec")): + if star == "*" and queue: + queue.pop(0) + if conv in "eEfFgGaA": + out.append("") + continue + value = queue.pop(0) if queue else 0 + if conv == "s": + out.append(ctypes.string_at(value).decode("utf-8", "replace") if value else "(null)") + elif conv == "c": + out.append(chr(value & 0xFF)) + elif conv == "p": + out.append(hex(value)) + elif conv in "di": + bits = 64 if match.group("len") in ("l", "ll", "j", "z", "t") else 32 + value &= (1 << bits) - 1 + out.append(str(value - (1 << bits) if value >> (bits - 1) else value)) + else: + bits = 64 if match.group("len") in ("l", "ll", "j", "z", "t") else 32 + value &= (1 << bits) - 1 + out.append({"o": f"{value:o}", "x": f"{value:x}", "X": f"{value:X}"}.get(conv, str(value))) + out.append(text[last:]) + return "".join(out).rstrip("\n") + + +def png_bytes(width: int, height: int, rgb: bytes) -> bytes: + def chunk(tag: bytes, body: bytes) -> bytes: + return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body)) + rows = b"".join(b"\0" + rgb[y * width * 3:(y + 1) * width * 3] for y in range(height)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(rows, 6)) + chunk(b"IEND", b"")) + + +def to_rgb(fmt: int, width: int, height: int, packed: bytes) -> bytes: + out = bytearray(width * height * 3) + if fmt == 1: + out[0::3], out[1::3], out[2::3] = packed[2::4], packed[1::4], packed[0::4] + return bytes(out) + for i, (pixel,) in enumerate(struct.iter_unpack("> 11) & 31, (pixel >> 5) & 63, pixel & 31 + out[i * 3:i * 3 + 3] = bytes((r << 3, g << 2, b << 3)) + else: + r, g, b = (pixel >> 10) & 31, (pixel >> 5) & 31, pixel & 31 + out[i * 3:i * 3 + 3] = bytes((r << 3, g << 3, b << 3)) + return bytes(out) + + +# -------------------------------------------------------------------------- +# libretro bindings (worker side) +# -------------------------------------------------------------------------- + +class RetroSystemInfo(ctypes.Structure): + _fields_ = [("library_name", ctypes.c_char_p), ("library_version", ctypes.c_char_p), + ("valid_extensions", ctypes.c_char_p), ("need_fullpath", ctypes.c_bool), + ("block_extract", ctypes.c_bool)] + + +class RetroGameGeometry(ctypes.Structure): + _fields_ = [("base_width", ctypes.c_uint), ("base_height", ctypes.c_uint), + ("max_width", ctypes.c_uint), ("max_height", ctypes.c_uint), + ("aspect_ratio", ctypes.c_float)] + + +class RetroSystemTiming(ctypes.Structure): + _fields_ = [("fps", ctypes.c_double), ("sample_rate", ctypes.c_double)] + + +class RetroSystemAvInfo(ctypes.Structure): + _fields_ = [("geometry", RetroGameGeometry), ("timing", RetroSystemTiming)] + + +class RetroGameInfo(ctypes.Structure): + _fields_ = [("path", ctypes.c_char_p), ("data", ctypes.c_void_p), + ("size", ctypes.c_size_t), ("meta", ctypes.c_char_p)] + + +class RetroVariable(ctypes.Structure): + _fields_ = [("key", ctypes.c_char_p), ("value", ctypes.c_char_p)] + + +LOG_ARGS = 16 +LogFn = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p, *([ctypes.c_size_t] * LOG_ARGS)) + + +class RetroLogCallback(ctypes.Structure): + _fields_ = [("log", LogFn)] + + +EnvFn = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_uint, ctypes.c_void_p) +VideoFn = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_uint, ctypes.c_uint, ctypes.c_size_t) +AudioFn = ctypes.CFUNCTYPE(None, ctypes.c_int16, ctypes.c_int16) +AudioBatchFn = ctypes.CFUNCTYPE(ctypes.c_size_t, ctypes.c_void_p, ctypes.c_size_t) +PollFn = ctypes.CFUNCTYPE(None) +InputFn = ctypes.CFUNCTYPE(ctypes.c_int16, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint, ctypes.c_uint) + + +class Frontend: + """The libretro callbacks for one run.""" + + def __init__(self, spec: dict) -> None: + self.spec = spec + self.system_dir = spec["system_dir"].encode() + self.save_dir = spec["save_dir"].encode() + self.options: dict[bytes, bytes] = {k.encode(): v.encode() for k, v in spec["options"].items()} + self.defaults: dict[bytes, bytes] = {} + self.declared: dict[str, list[str]] = {} + self.option_buffers: list[ctypes.Array] = [] + self.pixel_format = 0 + self.frame = 0 + self.last_frame = b"" + self.last_size = (0, 0) + self.video = hashlib.sha1() + self.audio = hashlib.sha1() + self.frame_hashes: list[list] = [] + self.logs: list[dict] = [] + self.presses: list[tuple[int, int, int]] = [tuple(p) for p in spec["presses"]] + self.unanswered: dict[int, int] = {} + self.input_queries = 0 + self.input_pressed = 0 + self.callbacks = ( + EnvFn(self.environment), VideoFn(self.video_refresh), AudioFn(self.audio_sample), + AudioBatchFn(self.audio_batch), PollFn(lambda: None), InputFn(self.input_state), + ) + self.log_callback = RetroLogCallback(LogFn(self.log)) + + def log(self, level: int, fmt: bytes, *args: int) -> None: + name = LOG_LEVELS[level] if 0 <= level < len(LOG_LEVELS) else str(level) + self.logs.append({"frame": self.frame, "level": name, "text": render_printf(fmt or b"", args)}) + + def environment(self, cmd: int, data: int) -> bool: + cmd &= ~ENV_EXPERIMENTAL + if cmd == ENV_GET_CAN_DUPE: + ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] = True + return True + if cmd in (ENV_GET_SYSTEM_DIRECTORY, ENV_GET_SAVE_DIRECTORY): + value = self.system_dir if cmd == ENV_GET_SYSTEM_DIRECTORY else self.save_dir + buffer = ctypes.create_string_buffer(value) + self.option_buffers.append(buffer) + ctypes.cast(data, ctypes.POINTER(ctypes.c_char_p))[0] = ctypes.cast(buffer, ctypes.c_char_p) + return True + if cmd == ENV_SET_PIXEL_FORMAT: + fmt = ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[0] + if fmt not in PIXEL_BYTES: + return False + self.pixel_format = fmt + return True + if cmd == ENV_SET_VARIABLES: + array = ctypes.cast(data, ctypes.POINTER(RetroVariable)) + index = 0 + while array[index].key: + key, desc = array[index].key, (array[index].value or b"").decode("utf-8", "replace") + values = desc.split(";", 1)[-1].strip().split("|") + self.declared[key.decode()] = values + self.defaults[key] = values[0].encode() + index += 1 + return True + if cmd == ENV_GET_VARIABLE: + var = ctypes.cast(data, ctypes.POINTER(RetroVariable))[0] + value = self.options.get(var.key, self.defaults.get(var.key)) + if value is None: + return False + buffer = ctypes.create_string_buffer(value) + self.option_buffers.append(buffer) + ctypes.cast(data, ctypes.POINTER(RetroVariable))[0].value = ctypes.cast(buffer, ctypes.c_char_p) + return True + if cmd == ENV_GET_VARIABLE_UPDATE: + ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[0] = False + return True + if cmd == ENV_SET_SUPPORT_NO_GAME: + return True + if cmd == ENV_GET_LOG_INTERFACE: + ctypes.cast(data, ctypes.POINTER(RetroLogCallback))[0] = self.log_callback + return True + if cmd == ENV_GET_CORE_OPTIONS_VERSION: + ctypes.cast(data, ctypes.POINTER(ctypes.c_uint))[0] = 0 + return True + self.unanswered[cmd] = self.unanswered.get(cmd, 0) + 1 + return False + + def video_refresh(self, data: int, width: int, height: int, pitch: int) -> None: + if data and data != ctypes.c_void_p(-1).value: + row = width * PIXEL_BYTES[self.pixel_format] + raw = ctypes.string_at(data, pitch * (height - 1) + row) if height else b"" + self.last_frame = b"".join(raw[y * pitch:y * pitch + row] for y in range(height)) + self.last_size = (width, height) + self.video.update(struct.pack(" None: + self.audio.update(struct.pack(" int: + self.audio.update(ctypes.string_at(data, frames * 4)) + return frames + + def input_state(self, port: int, device: int, index: int, button: int) -> int: + self.input_queries += 1 + if port or device != 1 or index: + return 0 + held = int(any(start <= self.frame < start + length and button == pressed + for start, pressed, length in self.presses)) + self.input_pressed += held + return held + + +def run_worker(spec: dict) -> dict: + """Load the core in this process and run it. Returns the measured result.""" + core = ctypes.CDLL(spec["core"]) + front = Frontend(spec) + for name, fn in zip(("retro_set_environment", "retro_set_video_refresh", "retro_set_audio_sample", + "retro_set_audio_sample_batch", "retro_set_input_poll", "retro_set_input_state"), + front.callbacks): + getattr(core, name)(fn) + info = RetroSystemInfo() + core.retro_get_system_info(ctypes.byref(info)) + core.retro_init() + game = None + if spec.get("content"): + path = spec["content"].encode() + game = RetroGameInfo(path, None, 0, None) + if not info.need_fullpath: + blob = Path(spec["content"]).read_bytes() + holder = ctypes.create_string_buffer(blob, len(blob)) + front.option_buffers.append(holder) + game.data, game.size = ctypes.cast(holder, ctypes.c_void_p), len(blob) + core.retro_load_game.restype = ctypes.c_bool + if not core.retro_load_game(ctypes.byref(game) if game else None): + return {"loaded": False, "logs": front.logs, "library_name": info.library_name.decode()} + # A frontend plugs a joypad into each port after loading, as RetroArch does. + for port in range(spec["joypad_ports"]): + core.retro_set_controller_port_device(ctypes.c_uint(port), ctypes.c_uint(1)) + av = RetroSystemAvInfo() + core.retro_get_system_av_info(ctypes.byref(av)) + warmup, frames = spec["warmup"], spec["frames"] + for _ in range(warmup): + core.retro_run() + front.frame += 1 + started = time.perf_counter() + for _ in range(frames): + core.retro_run() + front.frame += 1 + wall = time.perf_counter() - started + core.retro_serialize_size.restype = ctypes.c_size_t + size = core.retro_serialize_size() + state = b"" + if size: + buffer = ctypes.create_string_buffer(size) + core.retro_serialize.restype = ctypes.c_bool + if core.retro_serialize(buffer, ctypes.c_size_t(size)): + state = buffer.raw + if spec.get("state_out"): + Path(spec["state_out"]).write_bytes(state) + state_vars = mednafen_state_vars(state) + observables = { + "frame_sha1": hashlib.sha1(front.last_frame).hexdigest(), + "video_sha1": front.video.hexdigest(), + "audio_sha1": front.audio.hexdigest(), + "state_sha1": hashlib.sha1(state).hexdigest() if state else None, + } + for name in spec["state_vars"]: + observables[f"state:{name}"] = state_vars.get(name) + if spec["frame_hash_every"]: + observables["frame_hashes"] = front.frame_hashes + core.retro_unload_game() + core.retro_deinit() + return { + "loaded": True, + "library_name": info.library_name.decode(), + "library_version": info.library_version.decode(), + "fps_nominal": av.timing.fps, + "frames": frames, + "warmup": warmup, + "wall_s": wall, + "observables": observables, + "logs": front.logs, + "options_declared": front.declared, + "env_unanswered": {str(k): v for k, v in sorted(front.unanswered.items())}, + "last_frame_size": list(front.last_size), + "input_queries": front.input_queries, + "input_pressed": front.input_pressed, + } + + +# -------------------------------------------------------------------------- +# Host, core and frontend identity (parent side) +# -------------------------------------------------------------------------- + +def load_average() -> list[float] | None: + try: + return [round(value, 2) for value in os.getloadavg()] + except OSError: + return None + + +def cpu_model() -> str | None: + try: + for line in Path("/proc/cpuinfo").read_text(encoding="utf-8").splitlines(): + if line.startswith("model name"): + return line.split(":", 1)[1].strip() + except OSError: + return platform.processor() or None + return platform.processor() or None + + +def host_identity() -> dict: + return { + "cpu_model": cpu_model(), + "cpu_count": os.cpu_count(), + "os": platform.platform(), + "kernel": platform.release(), + "machine": platform.machine(), + "python": platform.python_version(), + } + + +def frontend_identity() -> dict: + script = Path(__file__).resolve() + commit: str | None = None + dirty: bool | None = None + try: + commit = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, + text=True, check=True).stdout.strip() + status = subprocess.run(["git", "status", "--porcelain", "--", str(script)], cwd=ROOT, + capture_output=True, text=True, check=True).stdout + dirty = bool(status.strip()) + except (OSError, subprocess.CalledProcessError) as exc: + print(f"WARNING: frontend commit unknown, git failed: {exc}", file=sys.stderr) + return {"script": "scripts/core_probe.py", "commit": commit, "script_modified": dirty, + "script_sha256": file_digest(script, "sha256")} + + +def core_identity(args: argparse.Namespace) -> dict: + core = Path(args.core) + return { + "file": core.name, + "sha256": file_digest(core, "sha256"), + "repo": args.core_repo, + "commit": args.core_commit, + "build_command": args.build_command, + "compiler": elf_comment(core) or None, + } + + +# -------------------------------------------------------------------------- +# Runs (parent side) +# -------------------------------------------------------------------------- + +@dataclass +class Variant: + label: str + files: dict[str, Path | None] = field(default_factory=dict) + + +def stage_system_dir(base: dict[str, Path], variant: Variant, target: Path) -> dict[str, str | None]: + """Copy the system files of one variant. Returns dest -> sha1 (None = absent).""" + target.mkdir(parents=True, exist_ok=True) + staged: dict[str, str | None] = {} + for dest, src in {**base, **variant.files}.items(): + out = target / dest + if src is None: + staged[dest] = None + continue + out.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, out) + staged[dest] = file_digest(out) + return staged + + +def run_once(args: argparse.Namespace, system_dir: Path, scratch: Path, state_out: Path | None, + png_dir: Path | None) -> dict: + save_dir = scratch / "save" + save_dir.mkdir(parents=True, exist_ok=True) + spec = { + "core": str(Path(args.core).resolve()), + "content": str(Path(args.content).resolve()) if args.content else None, + "system_dir": str(system_dir.resolve()), + "save_dir": str(save_dir.resolve()), + "options": dict(args.option), + "frames": args.frames, + "warmup": args.warmup, + "presses": [list(p) for p in args.press], + "state_vars": args.state_var, + "frame_hash_every": args.frame_hash_every, + "joypad_ports": args.joypad_ports, + "png_dir": str(png_dir) if png_dir else None, + "state_out": str(state_out) if state_out else None, + } + spec_path, out_path, trace_path = scratch / "spec.json", scratch / "result.json", scratch / "strace.txt" + spec_path.write_text(json.dumps(spec), encoding="utf-8") + command = [sys.executable, "-I", str(Path(__file__).resolve()), "_worker", str(spec_path), str(out_path)] + strace = shutil.which("strace") if not args.no_trace else None + if strace: + command = [strace, "-f", "-qq", "-s", "4096", "-e", "trace=openat,open", "-o", str(trace_path)] + command + env = {**os.environ, **dict(args.env)} + proc = subprocess.run(command, check=False, capture_output=True, text=True, + timeout=args.timeout, env=env) + if proc.returncode != 0 or not out_path.exists(): + raise ProbeError(f"worker exited {proc.returncode}: {proc.stderr.strip()[-2000:]}") + result = json.loads(out_path.read_text(encoding="utf-8")) + if strace: + opened, probed = files_under(parse_strace(trace_path.read_text(encoding="utf-8", errors="replace")), + system_dir) + result["files_opened"] = [{"path": rel, "sha1_after": file_digest(system_dir / rel)} + for rel in opened if (system_dir / rel).is_file()] + result["files_probed_absent"] = probed + result["tracing"] = "strace" + else: + result["files_opened"] = None + result["files_probed_absent"] = None + result["tracing"] = "unavailable: strace not found" if not args.no_trace else "disabled: --no-trace" + if result.get("wall_s") and result.get("frames"): + result["ms_per_frame"] = result["wall_s"] * 1000 / result["frames"] + result["fps"] = result["frames"] / result["wall_s"] + return result + + +def timing_stats(values: Sequence[float]) -> dict: + return { + "median": statistics.median(values), + "mean": statistics.fmean(values), + "stdev": statistics.stdev(values) if len(values) > 1 else 0.0, + "min": min(values), + "max": max(values), + "raw": list(values), + } + + +def run_variant(args: argparse.Namespace, base: dict[str, Path], variant: Variant, + work: Path, repetitions: int) -> dict: + system_dir = work / "system" + staged = stage_system_dir(base, variant, system_dir) + reps: list[dict] = [] + flags: list[str] = [] + for index in range(repetitions): + scratch = work / f"rep{index}" + scratch.mkdir(parents=True, exist_ok=True) + png_dir = None + if args.png_dir and index == 0: + png_dir = Path(args.png_dir) / variant.label + png_dir.mkdir(parents=True, exist_ok=True) + state_out = Path(args.state_dir) / f"{variant.label}.state" if args.state_dir and index == 0 else None + if state_out: + state_out.parent.mkdir(parents=True, exist_ok=True) + rep = run_once(args, system_dir, scratch, state_out, png_dir) + if not rep.get("loaded"): + flags.append(f"repetition {index}: retro_load_game returned false") + for entry in rep.get("files_opened") or []: + expected = staged.get(entry["path"]) + entry["sha1_staged"] = expected + if expected is not None and expected != entry["sha1_after"]: + flags.append(f"repetition {index}: {entry['path']} sha1 changed during the run") + reps.append(rep) + reference = reps[0].get("observables") + for index, rep in enumerate(reps[1:], 1): + if rep.get("observables") != reference: + differing = sorted(k for k in (reference or {}) if (rep.get("observables") or {}).get(k) != reference[k]) + flags.append(f"repetition {index} disagrees with repetition 0 on {', '.join(differing)}") + return { + "label": variant.label, + "system_files": {dest: sha for dest, sha in sorted(staged.items())}, + "repetitions": reps, + "flags": flags, + } + + +# -------------------------------------------------------------------------- +# Command line +# -------------------------------------------------------------------------- + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + fromfile_prefix_chars="@") + sub = parser.add_subparsers(dest="mode", required=True) + common = argparse.ArgumentParser(add_help=False, fromfile_prefix_chars="@") + common.add_argument("--core", required=True, help="libretro .so to load") + common.add_argument("--content", help="content to load (a .cue is identified with its tracks)") + common.add_argument("--content-label", help="identity of the content, e.g. its Redump name") + common.add_argument("--system-file", action="append", default=[], metavar="DEST=SRC", + type=lambda t: parse_assignment(t, "--system-file"), + help="copy SRC into the staged system dir as DEST") + common.add_argument("--option", action="append", default=[], metavar="KEY=VALUE", + type=lambda t: parse_assignment(t, "--option"), help="core option") + common.add_argument("--env", action="append", default=[], metavar="KEY=VALUE", + type=lambda t: parse_assignment(t, "--env"), + help="environment variable for the core process, e.g. LD_PRELOAD of a fixed clock") + common.add_argument("--frames", type=int, default=600, help="frames timed after warm-up") + common.add_argument("--warmup", type=int, default=0, help="frames run before timing starts") + common.add_argument("--press", action="append", default=[], type=parse_press, + metavar="FRAME:BUTTON[:LEN]", help="hold a joypad button on port 0") + common.add_argument("--joypad-ports", type=int, default=2, metavar="N", + help="ports given a joypad after loading (default 2)") + common.add_argument("--state-var", action="append", default=[], metavar="SECTION.VAR", + help="report a Mednafen save-state variable as an observable") + common.add_argument("--frame-hash-every", type=int, default=0, metavar="K", + help="record the sha1 of every K-th frame") + common.add_argument("--png-dir", help="write the frames sampled by --frame-hash-every as PNG") + common.add_argument("--state-dir", help="write the save state at the end of the first repetition") + common.add_argument("--core-repo", help="source repository URL of the core build") + common.add_argument("--core-commit", help="commit the core was built from") + common.add_argument("--build-command", help="command and flags that built the core") + common.add_argument("--record", metavar="DIR", help="write one JSON record under DIR//") + common.add_argument("--max-load", type=float, default=None, + help="refuse to record when the 1-minute load average exceeds this " + "(default: a quarter of the CPU count)") + common.add_argument("--work-dir", default=str(DEFAULT_WORK_DIR), help="scratch directory") + common.add_argument("--timeout", type=int, default=1800, help="seconds per repetition") + common.add_argument("--no-trace", action="store_true", help="do not run the worker under strace") + + sub.add_parser("run", parents=[common], help="one execution") + bench = sub.add_parser("bench", parents=[common], help="timed repetitions") + bench.add_argument("--repeat", type=int, default=MIN_BENCH_REPETITIONS) + check = sub.add_parser("check", parents=[common], help="compare variants of a system file") + check.add_argument("--vary", action="append", required=True, metavar="DEST=SRC", + type=lambda t: parse_assignment(t, "--vary"), + help="one variant per occurrence; an empty SRC leaves DEST absent") + check.add_argument("--observable", required=True, + help="the observable compared across variants") + check.add_argument("--claim", required=True, help="the statement under test") + check.add_argument("--method", required=True, help="how the run tests it") + check.add_argument("--repeat", type=int, default=2) + worker = sub.add_parser("_worker") + worker.add_argument("spec") + worker.add_argument("out") + return parser + + +def validate_args(args: argparse.Namespace) -> list[str]: + problems: list[str] = [] + if not Path(args.core).is_file(): + problems.append(f"--core {args.core}: not a file") + if args.content and not Path(args.content).is_file(): + problems.append(f"--content {args.content}: not a file") + for dest, src in args.system_file: + if not Path(src).is_file(): + problems.append(f"--system-file {dest}={src}: not a file") + if args.frames < 1 or args.warmup < 0: + problems.append("--frames must be >= 1 and --warmup >= 0") + if args.png_dir and not args.frame_hash_every: + problems.append("--png-dir needs --frame-hash-every") + if args.mode == "bench" and args.record and args.repeat < MIN_BENCH_REPETITIONS: + problems.append(f"a recorded benchmark needs --repeat >= {MIN_BENCH_REPETITIONS}") + if args.mode == "check": + dests = {dest for dest, _ in args.vary} + if len(dests) != 1: + problems.append("--vary must name the same DEST in every occurrence") + for dest, src in args.vary: + if src and not Path(src).is_file(): + problems.append(f"--vary {dest}={src}: not a file") + names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var} + if args.frame_hash_every: + names.add("frame_hashes") + if args.observable not in names: + problems.append(f"--observable {args.observable}: not one of {', '.join(sorted(names))}") + return problems + + +def variants_for(args: argparse.Namespace) -> list[Variant]: + if args.mode != "check": + return [Variant("default")] + out: list[Variant] = [] + seen: set[str] = set() + for dest, src in args.vary: + label = Path(src).name if src else f"{dest}-absent" + while label in seen: + label += "+" + seen.add(label) + out.append(Variant(label, {dest: Path(src) if src else None})) + return out + + +def check_result(variants: list[dict], observable: str) -> dict: + groups: dict[str, list[str]] = {} + failed = [v["label"] for v in variants if v["flags"]] + for variant in variants: + value = (variant["repetitions"][0].get("observables") or {}).get(observable) + groups.setdefault(json.dumps(value, sort_keys=True), []).append(variant["label"]) + if failed: + status = "inconclusive" + statement = (f"{len(failed)} of {len(variants)} variants carry flags " + f"({', '.join(failed)}); no comparison is drawn from them") + elif len(groups) == 1: + status = "same" + statement = f"{observable} is identical across all {len(variants)} variants" + else: + status = "differs" + statement = f"{observable} takes {len(groups)} distinct values across {len(variants)} variants" + return { + "status": status, + "statement": statement, + "groups": [{"value": json.loads(key), "variants": labels} for key, labels in groups.items()], + } + + +def record_path(directory: Path, core_name: str, record: dict) -> Path: + canonical = json.dumps(record, sort_keys=True, separators=(",", ":")).encode() + digest = hashlib.sha256(canonical).hexdigest()[:12] + date = record["recorded_at"][:10] + slug = re.sub(r"[^a-z0-9]+", "_", core_name.lower()).strip("_") or "core" + return directory / slug / f"{date}-{digest}.json" + + +def probe(args: argparse.Namespace) -> int: + problems = validate_args(args) + if problems: + for problem in problems: + print(f"ERROR: {problem}", file=sys.stderr) + return 2 + max_load = args.max_load if args.max_load is not None else (os.cpu_count() or 1) / 4 + load_before = load_average() + if args.record and load_before and load_before[0] > max_load: + print(f"ERROR: load average {load_before[0]} exceeds {max_load}: the machine is busy, " + f"nothing recorded (raise --max-load to override)", file=sys.stderr) + return 3 + repetitions = 1 if args.mode == "run" else args.repeat + base = {dest: Path(src) for dest, src in args.system_file} + work = Path(tempfile.mkdtemp(prefix=f"{args.mode}-", dir=_ensure(Path(args.work_dir)))) + try: + variants = [run_variant(args, base, variant, work / f"v{i}", repetitions) + for i, variant in enumerate(variants_for(args))] + except (ProbeError, subprocess.TimeoutExpired) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + finally: + shutil.rmtree(work, ignore_errors=True) + load_after = load_average() + first = variants[0]["repetitions"][0] + record: dict = { + "record_version": RECORD_VERSION, + "kind": args.mode, + "recorded_at": dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat(), + "core": {**core_identity(args), "library_name": first.get("library_name"), + "library_version": first.get("library_version")}, + "frontend": frontend_identity(), + "host": {**host_identity(), "load_before": load_before, "load_after": load_after, + "max_load": max_load}, + "inputs": { + "content": content_identity(Path(args.content), args.content_label) if args.content else None, + "options": dict(args.option), + "env": dict(args.env), + "frames": args.frames, + "warmup": args.warmup, + "repetitions": repetitions, + "presses": [{"frame": f, "button": b, "length": n} for f, b, n in args.press], + "observables": list(BASE_OBSERVABLES) + [f"state:{v}" for v in args.state_var] + + (["frame_hashes"] if args.frame_hash_every else []), + "frame_hash_every": args.frame_hash_every, + "joypad_ports": args.joypad_ports, + }, + "variants": variants, + "flags": [f"{v['label']}: {flag}" for v in variants for flag in v["flags"]], + } + if load_before and load_after and max(load_before[0], load_after[0]) > max_load: + record["flags"].append(f"load average reached {max(load_before[0], load_after[0])} " + f"(threshold {max_load})") + if args.mode == "bench": + reps = variants[0]["repetitions"] + record["timing"] = {"ms_per_frame": timing_stats([r["ms_per_frame"] for r in reps if "ms_per_frame" in r]), + "fps": timing_stats([r["fps"] for r in reps if "fps" in r])} + if args.mode == "check": + record["check"] = {"claim": args.claim, "method": args.method, "observable": args.observable, + "variable": args.vary[0][0], + "result": check_result(variants, args.observable)} + text = json.dumps(record, indent=2, sort_keys=False) + if args.record: + path = record_path(Path(args.record), first.get("library_name") or Path(args.core).stem, record) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text + "\n", encoding="utf-8") + print(f"recorded {path}", file=sys.stderr) + print(text) + return 0 + + +def _ensure(path: Path) -> Path: + path.mkdir(parents=True, exist_ok=True) + return path + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.mode == "_worker": + spec = json.loads(Path(args.spec).read_text(encoding="utf-8")) + Path(args.out).write_text(json.dumps(run_worker(spec)), encoding="utf-8") + return 0 + return probe(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 1a3b888c2..135b9f46b 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -616,6 +616,12 @@ def main() -> int: errors.extend(_save_directory_blocks()) errors.extend(_registry_contract()) errors.extend(_event_schema_errors()) + errors.extend( + _validate_json_files( + sorted((ROOT / "measurements").glob("**/*.json")), + "core-probe-record.schema.json", + ) + ) if not args.source_only: database_path = ROOT / "database.json" diff --git a/tests/test_core_probe.py b/tests/test_core_probe.py new file mode 100644 index 000000000..21cde243f --- /dev/null +++ b/tests/test_core_probe.py @@ -0,0 +1,269 @@ +"""core_probe runs a libretro core headless and records what it measured. + +The parsers are checked on fixed inputs. One test builds a ten-line +libretro core with the C compiler, when there is one, and runs the whole +probe on it: staging, the worker process, strace, the record and its schema. +""" + +from __future__ import annotations + +import argparse +import contextlib +import io +import json +import shutil +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import core_probe + +STRACE_SAMPLE = """\ +4242 openat(AT_FDCWD, "/sys/bios/card.bin", O_RDONLY) = 5 +4242 openat(AT_FDCWD, "/sys/bios/missing.bin", O_RDONLY) = -1 ENOENT (No such file or directory) +[pid 4243] open("/sys/bios/sub/x.rom", O_RDONLY|O_CLOEXEC) = 7 +4242 openat(AT_FDCWD, "/elsewhere/game.cue", O_RDONLY) = 3 +4242 read(3, "...", 10) = 10 +""" + +FAKE_CORE = r""" +#include +#include +#include +#include +#include +#include +struct retro_system_info { const char *n, *v, *e; bool full, block; }; +struct retro_game_geometry { unsigned bw, bh, mw, mh; float a; }; +struct retro_system_timing { double fps, rate; }; +struct retro_system_av_info { struct retro_game_geometry g; struct retro_system_timing t; }; +struct retro_log_callback { void (*log)(int, const char *, ...); }; +typedef bool (*env_t)(unsigned, void *); +typedef void (*video_t)(const void *, unsigned, unsigned, size_t); +typedef size_t (*batch_t)(const int16_t *, size_t); +typedef int16_t (*input_t)(unsigned, unsigned, unsigned, unsigned); +static env_t env; static video_t video; static batch_t batch; static input_t input; +static unsigned ports, frames; static struct retro_log_callback logger; +static uint16_t pixels[4 * 2]; static unsigned char card; +void retro_set_environment(env_t e) { bool t = true; env = e; env(18, &t); } +void retro_set_video_refresh(video_t v) { video = v; } +void retro_set_audio_sample(void *f) { (void)f; } +void retro_set_audio_sample_batch(batch_t b) { batch = b; } +void retro_set_input_poll(void *f) { (void)f; } +void retro_set_input_state(input_t f) { input = f; } +void retro_set_controller_port_device(unsigned port, unsigned device) { (void)port; if (device == 1) ports++; } +void retro_get_system_info(struct retro_system_info *i) { + i->n = "Fake"; i->v = "1"; i->e = ""; i->full = true; i->block = false; } +void retro_get_system_av_info(struct retro_system_av_info *i) { + memset(i, 0, sizeof *i); i->t.fps = 60.0; i->t.rate = 44100.0; } +void retro_init(void) {} +void retro_deinit(void) {} +bool retro_load_game(const void *g) { + const char *dir = NULL; char path[4096]; struct retro_log_callback cb; + const char *tag = getenv("CORE_PROBE_TEST_TAG"); + (void)g; env(9, &dir); + snprintf(path, sizeof path, "%s/card.bin", dir); + FILE *f = fopen(path, "rb"); + card = 0; + if (f) { int c = fgetc(f); card = c < 0 ? 0 : (unsigned char)c; fclose(f); } + if (env(27, &cb)) { + logger = cb; + cb.log(1, "card %s byte %d\n", f ? "present" : "absent", (int)card); + cb.log(1, "tag %s\n", tag ? tag : "unset"); } + return true; } +void retro_unload_game(void) {} +void retro_run(void) { + int16_t s[4] = {1, 2, 3, 4}; + if (frames++ == 0 && logger.log) logger.log(1, "ports %u\n", ports); + input(0, 1, 0, 8); + for (int i = 0; i < 8; i++) pixels[i] = card; + video(pixels, 4, 2, 8); batch(s, 2); } +size_t retro_serialize_size(void) { return 1; } +bool retro_serialize(void *d, size_t n) { (void)n; *(unsigned char *)d = card; return true; } +""" + + +def mednafen_state(sections: dict[str, dict[str, bytes]]) -> bytes: + body = b"" + for section, entries in sections.items(): + chunk = b"".join(bytes([len(name)]) + name.encode() + struct.pack(" None: + self.assertEqual(core_probe.parse_assignment("a=b=c", "--option"), ("a", "b=c")) + self.assertEqual(core_probe.parse_assignment("dest=", "--vary"), ("dest", "")) + for bad in ("novalue", "=x"): + with self.assertRaises(argparse.ArgumentTypeError): + core_probe.parse_assignment(bad, "--option") + + def test_press(self) -> None: + self.assertEqual(core_probe.parse_press("120:start"), (120, 3, 1)) + self.assertEqual(core_probe.parse_press("5:A:10"), (5, 8, 10)) + for bad in ("x:a", "1:turbo", "1:a:0", "-1:a", "1"): + with self.assertRaises(argparse.ArgumentTypeError): + core_probe.parse_press(bad) + + def test_strace(self) -> None: + calls = core_probe.parse_strace(STRACE_SAMPLE) + self.assertEqual([c["path"] for c in calls], + ["/sys/bios/card.bin", "/sys/bios/missing.bin", "/sys/bios/sub/x.rom", + "/elsewhere/game.cue"]) + self.assertEqual([c["ok"] for c in calls], [True, False, True, True]) + self.assertEqual(calls[1]["errno"], "ENOENT") + opened, absent = core_probe.files_under(calls, Path("/sys/bios")) + self.assertEqual(opened, ["card.bin", "sub/x.rom"]) + self.assertEqual(absent, ["missing.bin"]) + + def test_printf(self) -> None: + name = core_probe.ctypes.create_string_buffer(b"mpeg_card.bin") + address = core_probe.ctypes.addressof(name) + self.assertEqual(core_probe.render_printf(b"open %s: %d%% %x\n", [address, 50, 255]), + "open mpeg_card.bin: 50% ff") + self.assertEqual(core_probe.render_printf(b"%d", [0xFFFFFFFF]), "-1") + self.assertEqual(core_probe.render_printf(b"%.2f fps %u", [7]), " fps 7") + self.assertEqual(core_probe.render_printf(b"%s", [0]), "(null)") + + def test_mednafen_state(self) -> None: + state = b"\x00" * 7 + mednafen_state({"MPEG": {"Present": b"\x01", "AuthState": b"\x02"}, + "CDB": {"Word": b"\x34\x12"}}) + self.assertEqual(core_probe.mednafen_state_vars(state), + {"MPEG.Present": "01", "MPEG.AuthState": "02", "CDB.Word": "3412"}) + self.assertEqual(core_probe.mednafen_state_vars(b"no state"), {}) + + def test_elf_comment(self) -> None: + self.assertIsInstance(core_probe.elf_comment(Path(sys.executable).resolve()), list) + with tempfile.NamedTemporaryFile(dir=REPO_ROOT / "tmp" / "tests") as handle: + handle.write(b"not an elf") + handle.flush() + self.assertEqual(core_probe.elf_comment(Path(handle.name)), []) + + def test_cue_tracks(self) -> None: + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as tmp: + cue = Path(tmp, "d.cue") + cue.write_text('FILE "a b.bin" BINARY\n TRACK 01 MODE2/2352\nFILE c.bin BINARY\n', + encoding="utf-8") + self.assertEqual([p.name for p in core_probe.cue_tracks(cue)], ["a b.bin", "c.bin"]) + + +class ResultTests(unittest.TestCase): + @staticmethod + def variant(label: str, value: str, flags: list[str] | None = None) -> dict: + return {"label": label, "flags": flags or [], + "repetitions": [{"observables": {"video_sha1": value}}]} + + def test_same_differs_inconclusive(self) -> None: + same = core_probe.check_result([self.variant("a", "1"), self.variant("b", "1")], "video_sha1") + self.assertEqual(same["status"], "same") + differs = core_probe.check_result([self.variant("a", "1"), self.variant("b", "2")], "video_sha1") + self.assertEqual(differs["status"], "differs") + self.assertEqual(len(differs["groups"]), 2) + flagged = core_probe.check_result([self.variant("a", "1", ["x"]), self.variant("b", "1")], + "video_sha1") + self.assertEqual(flagged["status"], "inconclusive") + + def test_timing_stats(self) -> None: + stats = core_probe.timing_stats([1.0, 2.0, 3.0, 4.0, 10.0]) + self.assertEqual((stats["median"], stats["mean"], stats["min"], stats["max"]), (3.0, 4.0, 1.0, 10.0)) + self.assertEqual(stats["raw"], [1.0, 2.0, 3.0, 4.0, 10.0]) + + def test_refusals(self) -> None: + parser = core_probe.build_parser() + core = sys.executable + bench = parser.parse_args(["bench", "--core", core, "--record", "x", "--repeat", "3"]) + self.assertIn("a recorded benchmark needs --repeat >= 5", core_probe.validate_args(bench)) + check = parser.parse_args(["check", "--core", core, "--vary", "a=", "--vary", "b=", + "--observable", "video_sha1", "--claim", "c", "--method", "m"]) + self.assertTrue(any("same DEST" in p for p in core_probe.validate_args(check))) + unknown = parser.parse_args(["check", "--core", core, "--vary", "a=", "--observable", "nope", + "--claim", "c", "--method", "m"]) + self.assertTrue(any("--observable nope" in p for p in core_probe.validate_args(unknown))) + + +@unittest.skipUnless(shutil.which("cc"), "no C compiler") +class FakeCoreTests(unittest.TestCase): + """The whole probe against a core compiled from FAKE_CORE.""" + + @classmethod + def setUpClass(cls) -> None: + cls.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests")) + source = cls.tmp / "fake.c" + source.write_text(FAKE_CORE, encoding="utf-8") + cls.core = cls.tmp / "fake_libretro.so" + subprocess.run(["cc", "-shared", "-fPIC", "-o", str(cls.core), str(source)], check=True) + cls.cards = [] + for byte in (b"\x01", b"\x01", b"\x02"): + path = cls.tmp / f"card{len(cls.cards)}.bin" + path.write_bytes(byte) + cls.cards.append(path) + + @classmethod + def tearDownClass(cls) -> None: + shutil.rmtree(cls.tmp, ignore_errors=True) + + def probe(self, argv: list[str]) -> tuple[int, dict | None]: + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + code = core_probe.main(argv + ["--work-dir", str(self.tmp / "work")]) + return code, json.loads(out.getvalue()) if code == 0 else None + + def test_run_record(self) -> None: + record_dir = self.tmp / "records" + code, record = self.probe(["check", "--core", str(self.core), "--frames", "3", + "--vary", f"card.bin={self.cards[0]}", "--vary", f"card.bin={self.cards[1]}", + "--vary", f"card.bin={self.cards[2]}", "--vary", "card.bin=", + "--observable", "frame_sha1", "--claim", "c", "--method", "m", + "--record", str(record_dir), "--max-load", "1000", + "--env", "CORE_PROBE_TEST_TAG=set-by-test", "--press", "0:a:2"]) + self.assertEqual(code, 0) + result = record["check"]["result"] + self.assertEqual(result["status"], "differs") + self.assertEqual(sorted(len(g["variants"]) for g in result["groups"]), [1, 1, 2]) + first = record["variants"][0]["repetitions"][0] + self.assertEqual(first["logs"][0]["text"], "card present byte 1") + self.assertEqual(first["logs"][1]["text"], "tag set-by-test") + self.assertEqual(first["logs"][2]["text"], "ports 2") + self.assertEqual((first["input_queries"], first["input_pressed"]), (3, 2)) + self.assertEqual(record["inputs"]["env"], {"CORE_PROBE_TEST_TAG": "set-by-test"}) + self.assertEqual(record["variants"][3]["repetitions"][0]["logs"][0]["text"], "card absent byte 0") + if first["tracing"] == "strace": + self.assertEqual([f["path"] for f in first["files_opened"]], ["card.bin"]) + self.assertEqual(record["variants"][3]["repetitions"][0]["files_probed_absent"], ["card.bin"]) + self.assertEqual(first["frames"], 3) + written = list(record_dir.glob("*/*.json")) + self.assertEqual(len(written), 1) + self.assertEqual(written[0].parent.name, "fake") + try: + from jsonschema import Draft202012Validator + except ImportError: + return + schema = json.loads((REPO_ROOT / "schemas" / "core-probe-record.schema.json").read_text()) + errors = list(Draft202012Validator(schema).iter_errors(json.loads(written[0].read_text()))) + self.assertEqual(errors, []) + + def test_bench_stats(self) -> None: + code, record = self.probe(["bench", "--core", str(self.core), "--frames", "2", "--warmup", "1", + "--repeat", "5", "--no-trace", "--max-load", "1000"]) + self.assertEqual(code, 0) + self.assertEqual(len(record["timing"]["ms_per_frame"]["raw"]), 5) + self.assertEqual(record["variants"][0]["repetitions"][0]["tracing"], "disabled: --no-trace") + self.assertEqual(record["flags"], []) + + def test_busy_machine_refused(self) -> None: + code, _ = self.probe(["run", "--core", str(self.core), "--frames", "1", "--record", + str(self.tmp / "refused"), "--max-load", "-1"]) + self.assertEqual(code, 3) + self.assertFalse((self.tmp / "refused").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 9091f6571..1ef9353c9 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -56,6 +56,7 @@ Every module in `tests/`, by the area it guards. | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_app`, `test_install_from`, `test_install_offline`, `test_install_selection`, `test_install_tui` | | Handheld launchers | a pty as the console, fake installer and firmware tools, stubbed PortMaster dialogs | `test_handheld_launchers`, `test_port` | +| Running cores | parsers on fixed inputs, a libretro core built from C in the test when a compiler exists | `test_core_probe` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_availability`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_picker`, `test_site_platform_hashes`, `test_site_r2_components`, `test_site_r2_css`, `test_site_r2_emulators`, `test_site_r2_entry`, `test_site_r2_gate`, `test_site_r2_nav`, `test_site_r2_p0`, `test_site_r2_reference`, `test_site_r2_systems`, `test_site_r2_wiki`, `test_site_r3_collection`, `test_site_r3_components`, `test_site_r3_css`, `test_site_r3_emulators`, `test_site_r3_entry`, `test_site_r3_nav`, `test_site_r3_p0`, `test_site_r3_reference`, `test_site_r3_systems`, `test_site_r3_wiki`, `test_site_r4_components`, `test_site_r4_css`, `test_site_r4_emulators`, `test_site_r4_entry`, `test_site_r4_merge`, `test_site_r4_p0`, `test_site_r4_reference`, `test_site_r4_search`, `test_site_r4_systems`, `test_site_r4_wiki`, `test_site_r5_components`, `test_site_r5_css`, `test_site_r5_emulators`, `test_site_r5_entry`, `test_site_r5_reference`, `test_site_r5_systems`, `test_site_r5_wiki`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_weight`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | diff --git a/wiki/tools.md b/wiki/tools.md index e3e35f76a..ad6c98dc2 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -704,6 +704,60 @@ python scripts/refresh_stale.py python scripts/refresh_stale.py --only platforms,targets --jobs 6 ``` +### core_probe.py + +Runs a libretro core with no window, no sound and no input device, and +reports what it did. It is the test in real conditions behind a claim about +a core: which files it opens, what it shows, how fast it runs. + +```bash +# one run, printed as JSON +python scripts/core_probe.py run --core mednafen_saturn_libretro.so \ + --content game.cue --system-file sega_101.bin=bios/Sega/Saturn/sega_101.bin \ + --option beetle_saturn_region=Japan --frames 600 + +# timed repetitions after warm-up, written as a record +python scripts/core_probe.py bench --core core.so --content game.cue \ + --warmup 300 --frames 3000 --repeat 5 --record measurements/ + +# one system file swapped between runs, compared on one observable +python scripts/core_probe.py check --core core.so --content disc.cue \ + --vary mpeg_card.bin=a.bin --vary mpeg_card.bin=b.bin --vary mpeg_card.bin= \ + --observable video_sha1 --claim "..." --method "..." --record measurements/ +``` + +Each repetition runs in a fresh process under `strace -f -e trace=openat,open` +when strace is installed. The files the core opened under the system +directory come from the kernel, each hashed again after the run. Without +strace the record says that tracing was unavailable. + +| Option | Effect | +|--------|--------| +| `--system-file DEST=SRC` | copy SRC into a staged system directory as DEST | +| `--option KEY=VALUE` | core option. Unset options take the first value the core declares | +| `--frames`, `--warmup` | frames timed, and frames run before timing starts | +| `--press FRAME:BUTTON[:LEN]` | hold a libretro joypad button on port 0 (`a`, `b`, `start`, `up`...). A menu may need 15 frames or more to see it | +| `--joypad-ports N` | ports given a joypad after loading, as RetroArch does (default 2) | +| `--env KEY=VALUE` | environment of the core process. A core that reads the host clock repeats only with the clock fixed, for example `LD_PRELOAD` of libfaketime with `FAKETIME` | +| `--frame-hash-every K`, `--png-dir` | sha1 of every K-th frame, and those frames as PNG | +| `--state-var SECTION.VAR` | a variable of a Mednafen save state, as an observable | +| `--state-dir` | keep the save state taken at the end of the first repetition | +| `--core-repo`, `--core-commit`, `--build-command` | where the core build comes from, copied into the record | +| `--record DIR` | write `DIR//-.json` (schema `core-probe-record.schema.json`) | +| `--max-load` | refuse to record when the 1-minute load average is above it (default: a quarter of the CPU count) | +| `--no-trace` | run without strace | +| `@FILE` | read arguments from FILE, one per line | + +Observables are `frame_sha1` (last frame), `video_sha1` (every frame, chained), +`audio_sha1`, `state_sha1` (save state at the end), plus `state:SECTION.VAR` +and `frame_hashes` when asked for. A repetition that disagrees with the first +one is flagged, and a `check` with a flagged variant reports `inconclusive` +instead of a comparison. A record carries the core build (sha256 of the +`.so`, the compilers its `.comment` section names), the frontend commit, the +host (CPU, kernel, Python, load before and after) and each input by hash. +Core log lines arrive through the libretro log interface. Floating-point +arguments of a log line cannot be read from Python and print as ``. + ### Other tools | Script | Purpose |