mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-01 00:27:20 -05:00
feat: add the real-condition campaign runner
campaign.py takes buildbot nightlies (or a build of the pinned commit), the test content of tests/campaign/content.yml through romfetch, and runs core_probe for the files, boot, path, alias, variants and bench checks. A fixed rule table settles hle_fallback, required, path and aliases with the record ids as evidence, the rest goes to measurements/findings.jsonl. Resumable state, load, disk and budget guards, retries, content reference counting with an LRU cap, and an autorun job after the nightly.
This commit is contained in:
Abdessamad Derraz committed
1 parent
faa79c5920
commit
ee7319d838
16 files changed
+2990
-11
No files matched your search
@@ -85,6 +85,32 @@ jobs:
|
||||
stdout: tmp/autorun/bios-integrity.json
|
||||
report: tmp/autorun/bios-integrity.json
|
||||
|
||||
campaign:
|
||||
schedule: "*-*-* 06:00"
|
||||
network: true
|
||||
max_load_wait: 7200
|
||||
timeout: 12600
|
||||
steps:
|
||||
- run: [python, scripts/campaign.py, run, --budget, 150m]
|
||||
ok_codes: [0, 75]
|
||||
stdout: tmp/autorun/campaign.json
|
||||
gate:
|
||||
- [python, scripts/campaign.py, gate]
|
||||
commits:
|
||||
- subject: "chore: add campaign measurements"
|
||||
body: >-
|
||||
Core probe records, findings and the pass summary written by
|
||||
campaign.py, behind the measurement tests and the schemas.
|
||||
paths: [measurements/*.jsonl]
|
||||
- subject: "chore: apply fields the campaign proved"
|
||||
body: >-
|
||||
Fields the campaign's rule table settled from real runs, each with
|
||||
the record ids in evidence:, behind verify and an emulator pack of
|
||||
every changed profile.
|
||||
paths: [emulators/*.yml]
|
||||
report: tmp/autorun/campaign.json
|
||||
scratch: [tmp/autorun, tmp/core_probe]
|
||||
|
||||
assets:
|
||||
schedule: "*-*-* 04:30"
|
||||
network: true
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/campaign-finding.schema.json",
|
||||
"title": "RetroBIOS campaign finding",
|
||||
"description": "One line of measurements/findings.jsonl: what a real-condition run showed about one field of one profile entry, the rule that settled it or the reason it waits for review, and the records that show it. Lines are appended, never rewritten: the latest line of a key is its state.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_version", "id", "found_at", "key", "emulator", "system", "file", "field",
|
||||
"status", "rule", "profile_says", "run_shows", "statement", "evidence", "core", "content"],
|
||||
"properties": {
|
||||
"schema_version": { "const": 1 },
|
||||
"id": {
|
||||
"type": "string",
|
||||
"pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$",
|
||||
"description": "Date and the first 12 hex digits of the sha256 of the line without its id, in canonical JSON."
|
||||
},
|
||||
"found_at": { "type": "string", "format": "date-time" },
|
||||
"key": {
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"description": "emulator/system/file/field: the lines of one key are the history of one claim."
|
||||
},
|
||||
"emulator": { "type": "string", "pattern": "^[a-z0-9]+(?:[-_][a-z0-9]+)*$" },
|
||||
"system": { "type": "string", "minLength": 1 },
|
||||
"file": { "type": "string", "minLength": 1 },
|
||||
"field": { "type": "string", "minLength": 1 },
|
||||
"status": {
|
||||
"enum": ["applied", "open", "flagged", "resolved"],
|
||||
"description": "applied: the rule table settled the field and the profile was changed. open: no rule settles it, it waits for review. flagged: a declared file the runs never opened. resolved: a later run found the profile in agreement."
|
||||
},
|
||||
"rule": { "type": "string", "minLength": 1 },
|
||||
"profile_says": {},
|
||||
"run_shows": {},
|
||||
"statement": { "type": "string", "minLength": 1 },
|
||||
"evidence": {
|
||||
"type": "array",
|
||||
"minItems": 1,
|
||||
"items": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$" },
|
||||
"description": "Ids of the records in measurements/<emulator>.jsonl that show it."
|
||||
},
|
||||
"core": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["file", "sha256", "library_version"],
|
||||
"properties": {
|
||||
"file": { "type": "string", "minLength": 1 },
|
||||
"sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
|
||||
"library_version": { "type": ["string", "null"] }
|
||||
}
|
||||
},
|
||||
"content": { "type": ["string", "null"] }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://abdess.github.io/retrobios/api/v1/schemas/campaign-run.schema.json",
|
||||
"title": "RetroBIOS campaign run",
|
||||
"description": "One line of measurements/campaign-runs.jsonl: one pass of scripts/campaign.py, what it left done, failed and pending, and the records and findings it added. Lines are appended, never rewritten.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["schema_version", "id", "started_at", "ended_at", "result", "detail", "counts",
|
||||
"pending_reasons", "failed", "records", "findings"],
|
||||
"properties": {
|
||||
"schema_version": { "const": 1 },
|
||||
"id": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$" },
|
||||
"started_at": { "type": "string", "format": "date-time" },
|
||||
"ended_at": { "type": "string", "format": "date-time" },
|
||||
"result": {
|
||||
"enum": ["complete", "budget", "paused", "stopped"],
|
||||
"description": "complete: nothing runnable was left. budget: the time budget ended the pass. paused: a busy host or a full disk ended it, the next pass resumes. stopped: interrupted."
|
||||
},
|
||||
"detail": { "type": "string" },
|
||||
"counts": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["done", "failed", "pending"],
|
||||
"properties": {
|
||||
"done": { "type": "integer", "minimum": 0 },
|
||||
"failed": { "type": "integer", "minimum": 0 },
|
||||
"pending": { "type": "integer", "minimum": 0 }
|
||||
}
|
||||
},
|
||||
"pending_reasons": {
|
||||
"type": "object",
|
||||
"additionalProperties": { "type": "integer", "minimum": 1 },
|
||||
"description": "Pending checks counted by reason."
|
||||
},
|
||||
"failed": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": ["emulator", "check", "reason"],
|
||||
"properties": {
|
||||
"emulator": { "type": "string" },
|
||||
"check": { "type": "string" },
|
||||
"reason": { "type": "string" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"records": {
|
||||
"type": "array",
|
||||
"items": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$" }
|
||||
},
|
||||
"findings": {
|
||||
"type": "object",
|
||||
"additionalProperties": { "type": "integer", "minimum": 0 },
|
||||
"description": "Finding lines added by this pass, counted by status."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -125,6 +125,11 @@
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"artifact": {
|
||||
"type": "string",
|
||||
"format": "uri",
|
||||
"description": "Where the .so was downloaded from, such as a libretro buildbot nightly. Absent for a core built locally."
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
@@ -478,6 +478,18 @@
|
||||
"hle_fallback": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"evidence": {
|
||||
"description": "Real-condition runs that settled a field of this entry, by field: the ids of the core probe records in measurements/<emulator>.jsonl. Written by scripts/campaign.py when its rule table settles the field.",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"minProperties": 1,
|
||||
"properties": {
|
||||
"required": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$"}},
|
||||
"hle_fallback": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$"}},
|
||||
"path": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$"}},
|
||||
"aliases": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}-[0-9a-f]{12}$"}}
|
||||
}
|
||||
},
|
||||
"adler32_byteswap": {
|
||||
"type": "boolean"
|
||||
},
|
||||
|
||||
@@ -54,6 +54,7 @@ JOBS_FILE = REPO_ROOT / "automation" / "jobs.yml"
|
||||
AUTORUN_DIR = Path(".cache") / "autorun"
|
||||
DEFAULT_LOCK = Path.home() / ".local" / "state" / "retrobios" / "auto.lock"
|
||||
LOCK_ENV = "RETROBIOS_AUTO_LOCK"
|
||||
LOCK_HELD_ENV = "RETROBIOS_AUTO_LOCK_HELD"
|
||||
|
||||
BRANCH = "autorun"
|
||||
REMOTE = "origin"
|
||||
@@ -726,6 +727,9 @@ def set_alert(state: dict, name: str, active: bool, detail: str, now: str) -> No
|
||||
def job_env(job: Job) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env["RETROBIOS_AUTO"] = "1"
|
||||
if job.lock:
|
||||
# A step that takes the host lock itself (campaign.py) runs under ours.
|
||||
env[LOCK_HELD_ENV] = "1"
|
||||
if job.network:
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
from common import github_token
|
||||
|
||||
+1511
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,161 @@
|
||||
"""Write a settled field into one entry of a profile, and nothing else.
|
||||
|
||||
The profile is edited as text, so its comments, order and quoting stay as
|
||||
the maintainer wrote them. Every edit is checked by parsing the result: the
|
||||
edited entry must equal the old one with the new fields, and every other
|
||||
part of the profile must parse to what it did before. An edit that cannot
|
||||
hold both is refused (ProfileEditError), never written.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ITEM = re.compile(r"^(?P<indent>\s*)- ")
|
||||
KEY = re.compile(r"^(?P<indent>\s*)(?P<key>[A-Za-z_][A-Za-z0-9_]*):(?P<rest>.*)$")
|
||||
# Where a new key goes: after the first of these the entry has, else last.
|
||||
ANCHORS = {
|
||||
"required": ("name",),
|
||||
"hle_fallback": ("required", "name"),
|
||||
"path": ("name",),
|
||||
"aliases": ("path", "name"),
|
||||
}
|
||||
|
||||
|
||||
class ProfileEditError(ValueError):
|
||||
"""An edit the profile's layout does not allow to make safely."""
|
||||
|
||||
|
||||
def scalar(value: object) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if value is None:
|
||||
return "null"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
if isinstance(value, str):
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
if isinstance(value, list):
|
||||
return "[" + ", ".join(scalar(v) for v in value) + "]"
|
||||
raise ProfileEditError(f"no scalar form for {value!r}")
|
||||
|
||||
|
||||
def item_spans(lines: list[str]) -> list[tuple[int, int, int]]:
|
||||
"""(first line, end line, item indent) of each item of the top-level files: list."""
|
||||
try:
|
||||
start = next(i for i, line in enumerate(lines) if line.rstrip() == "files:")
|
||||
except StopIteration as exc:
|
||||
raise ProfileEditError("no top-level files: list") from exc
|
||||
spans: list[tuple[int, int, int]] = []
|
||||
indent: int | None = None
|
||||
current: int | None = None
|
||||
end = len(lines)
|
||||
for i in range(start + 1, len(lines)):
|
||||
line = lines[i]
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
width = len(line) - len(line.lstrip(" "))
|
||||
if width == 0 and not line.startswith("-"):
|
||||
end = i
|
||||
break
|
||||
match = ITEM.match(line)
|
||||
if match and (indent is None or width == indent):
|
||||
indent = width
|
||||
if current is not None:
|
||||
spans.append((current, i, indent))
|
||||
current = i
|
||||
if current is not None:
|
||||
spans.append((current, end, indent or 0))
|
||||
# An item ends at its last line of content: the comments and blank lines
|
||||
# before the next item belong to that item.
|
||||
trimmed = []
|
||||
for first, stop, width in spans:
|
||||
last = stop
|
||||
while last > first + 1 and (not lines[last - 1].strip() or lines[last - 1].strip().startswith("#")):
|
||||
last -= 1
|
||||
trimmed.append((first, last, width))
|
||||
return trimmed
|
||||
|
||||
|
||||
def key_lines(lines: list[str], first: int, stop: int, key_indent: int) -> dict[str, tuple[int, int]]:
|
||||
"""key -> (line, end line) for each key of one item, its continuation lines included."""
|
||||
keys: dict[str, tuple[int, int]] = {}
|
||||
order: list[tuple[str, int]] = []
|
||||
for i in range(first, stop):
|
||||
# The item's first line carries the dash where its first key's indent is.
|
||||
line = " " * key_indent + lines[i][key_indent:] if i == first else lines[i]
|
||||
match = KEY.match(line)
|
||||
if match and len(match["indent"]) == key_indent:
|
||||
order.append((match["key"], i))
|
||||
for position, (key, line) in enumerate(order):
|
||||
end = order[position + 1][1] if position + 1 < len(order) else stop
|
||||
while end > line + 1 and (not lines[end - 1].strip() or lines[end - 1].strip().startswith("#")):
|
||||
end -= 1
|
||||
keys[key] = (line, end)
|
||||
return keys
|
||||
|
||||
|
||||
def _render(key: str, value: object, indent: int, block: bool = False) -> list[str]:
|
||||
"""One key as lines. A list keeps the block style it was written in."""
|
||||
pad = " " * indent
|
||||
if isinstance(value, dict):
|
||||
return [f"{pad}{key}:"] + [f"{pad} {k}: {scalar(v)}" for k, v in value.items()]
|
||||
if block and isinstance(value, list):
|
||||
return [f"{pad}{key}:"] + [f"{pad} - {scalar(v)}" for v in value]
|
||||
return [f"{pad}{key}: {scalar(value)}"]
|
||||
|
||||
|
||||
def set_fields(text: str, index: int, updates: dict[str, object]) -> str:
|
||||
"""The profile with these keys of files[index] set, the rest untouched."""
|
||||
before = yaml.safe_load(text)
|
||||
entries = before.get("files") or []
|
||||
if not 0 <= index < len(entries):
|
||||
raise ProfileEditError(f"no files[{index}]")
|
||||
lines = text.splitlines()
|
||||
spans = item_spans(lines)
|
||||
if len(spans) != len(entries):
|
||||
raise ProfileEditError(f"{len(spans)} items read as text, {len(entries)} parsed")
|
||||
first, stop, item_indent = spans[index]
|
||||
key_indent = item_indent + 2
|
||||
for key, value in updates.items():
|
||||
keys = key_lines(lines, first, stop, key_indent)
|
||||
if key in keys:
|
||||
start, end = keys[key]
|
||||
rendered = _render(key, value, key_indent, block=lines[start].rstrip().endswith(":"))
|
||||
if start == first:
|
||||
raise ProfileEditError(f"files[{index}]: {key} is the item's first key")
|
||||
lines[start:end] = rendered
|
||||
stop += len(rendered) - (end - start)
|
||||
continue
|
||||
rendered = _render(key, value, key_indent)
|
||||
anchor = next((keys[a][1] for a in ANCHORS.get(key, ()) if a in keys), None)
|
||||
at = anchor if anchor is not None else stop
|
||||
lines[at:at] = rendered
|
||||
stop += len(rendered)
|
||||
out = "\n".join(lines) + ("\n" if text.endswith("\n") else "")
|
||||
after = yaml.safe_load(out)
|
||||
expected = {**entries[index], **updates}
|
||||
if after.get("files", [None] * (index + 1))[index] != expected:
|
||||
raise ProfileEditError(f"files[{index}]: the edit does not parse to the intended entry")
|
||||
rest_before = {k: v for k, v in before.items() if k != "files"}
|
||||
rest_after = {k: v for k, v in after.items() if k != "files"}
|
||||
others_before = [e for i, e in enumerate(entries) if i != index]
|
||||
others_after = [e for i, e in enumerate(after.get("files") or []) if i != index]
|
||||
if rest_before != rest_after or others_before != others_after:
|
||||
raise ProfileEditError(f"files[{index}]: the edit changes another part of the profile")
|
||||
return out
|
||||
|
||||
|
||||
def write_fields(path: Path, index: int, updates: dict[str, object]) -> bool:
|
||||
"""Apply the edit to the file. False when it changes nothing."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
out = set_fields(text, index, updates)
|
||||
if out == text:
|
||||
return False
|
||||
path.write_text(out, encoding="utf-8")
|
||||
return True
|
||||
@@ -0,0 +1,203 @@
|
||||
"""The campaign's fixed rule table: which observable settles which profile field.
|
||||
|
||||
Each rule is a pure function of a core probe record (or what was read from
|
||||
one) and a profile entry. It returns verdicts. A verdict with `apply` set is
|
||||
settled by the table and written into the profile with the record ids as
|
||||
`evidence:`. Any other verdict goes to the review queue
|
||||
(measurements/findings.jsonl).
|
||||
|
||||
observable field settled
|
||||
boots to the game with no file hle_fallback true
|
||||
no game screen without it, the game with it required true
|
||||
hle_fallback false
|
||||
opens the staged file under another path path that path
|
||||
opens the staged file under another name aliases name added
|
||||
a staged file the content should use, never read - flagged only
|
||||
|
||||
"Boots to the game" is read from the sampled frames: the run without the
|
||||
file shows at least MIN_SHARED_SCREENS distinct screens that the run with
|
||||
the file also shows, single-colour frames excluded. "No game screen" is a
|
||||
run that did not load, or whose every sampled frame is a single colour. A
|
||||
run that shows screens of its own that the reference never shows settles
|
||||
nothing.
|
||||
|
||||
Standard library only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
MIN_SHARED_SCREENS = 2
|
||||
BOOTS = "boots"
|
||||
FAILS = "fails"
|
||||
INCONCLUSIVE = "inconclusive"
|
||||
|
||||
# A field an entry leaves out reads as this value.
|
||||
FIELD_DEFAULTS: dict[str, object] = {"hle_fallback": False, "path": None, "aliases": []}
|
||||
SETTLED_FIELDS = ("required", "hle_fallback", "path", "aliases")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Boot:
|
||||
"""What the runs without the file showed, against the runs with it."""
|
||||
|
||||
status: str
|
||||
detail: str
|
||||
shared: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Verdict:
|
||||
rule: str
|
||||
field: str
|
||||
value: object
|
||||
statement: str
|
||||
evidence: tuple[str, ...]
|
||||
apply: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Opened:
|
||||
"""Where a run found the staged file opened."""
|
||||
|
||||
dest: str
|
||||
record_id: str
|
||||
extra: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def screens(repetition: dict) -> set[str]:
|
||||
"""The distinct sampled frames of a repetition that are not one colour."""
|
||||
observables = repetition.get("observables") or {}
|
||||
uniform = set(observables.get("uniform_frames") or ())
|
||||
return {digest for _, digest in observables.get("frame_hashes") or () if digest not in uniform}
|
||||
|
||||
|
||||
def variant(record: dict, label: str) -> dict:
|
||||
for item in record.get("variants") or ():
|
||||
if item.get("label") == label:
|
||||
return item
|
||||
raise KeyError(f"record {record.get('id')}: no variant {label!r}")
|
||||
|
||||
|
||||
def boot_outcome(record: dict, absent_label: str, present_label: str) -> Boot:
|
||||
"""Whether the core reached the game without the file.
|
||||
|
||||
Every repetition must agree: one that boots and one that does not leave
|
||||
the question open.
|
||||
"""
|
||||
present = variant(record, present_label)["repetitions"]
|
||||
if any(not rep.get("loaded") for rep in present):
|
||||
return Boot(INCONCLUSIVE, "with the file, the core did not load the content")
|
||||
reference: set[str] = set()
|
||||
for rep in present:
|
||||
shown = screens(rep)
|
||||
if not shown:
|
||||
return Boot(INCONCLUSIVE, "with the file, the core showed no game screen")
|
||||
reference |= shown
|
||||
outcomes: list[str] = []
|
||||
shared_counts: list[int] = []
|
||||
for rep in variant(record, absent_label)["repetitions"]:
|
||||
if not rep.get("loaded"):
|
||||
outcomes.append(FAILS)
|
||||
continue
|
||||
shown = screens(rep)
|
||||
if not shown:
|
||||
outcomes.append(FAILS)
|
||||
continue
|
||||
shared = len(shown & reference)
|
||||
shared_counts.append(shared)
|
||||
outcomes.append(BOOTS if shared >= MIN_SHARED_SCREENS else INCONCLUSIVE)
|
||||
shared = min(shared_counts) if shared_counts else 0
|
||||
if all(o == BOOTS for o in outcomes):
|
||||
return Boot(BOOTS, f"without the file, every run showed {shared} or more screens the "
|
||||
f"run with the file shows", shared)
|
||||
if all(o == FAILS for o in outcomes):
|
||||
return Boot(FAILS, "without the file, no run loaded the content or showed more than "
|
||||
"one colour, while the run with the file reached the game")
|
||||
return Boot(INCONCLUSIVE, "without the file, the runs showed screens the run with the file "
|
||||
"never shows, or disagreed with each other", shared)
|
||||
|
||||
|
||||
def current(entry: dict, name: str) -> object:
|
||||
if name in entry:
|
||||
return entry[name]
|
||||
return FIELD_DEFAULTS.get(name)
|
||||
|
||||
|
||||
def agrees(entry: dict, verdict: Verdict) -> bool:
|
||||
"""True when the profile already says what the run showed."""
|
||||
if verdict.field == "aliases":
|
||||
return set(verdict.value or ()) <= set(entry.get("aliases") or ())
|
||||
return current(entry, verdict.field) == verdict.value
|
||||
|
||||
|
||||
def rule_bios_need(boot: Boot, entry: dict, record_id: str) -> list[Verdict]:
|
||||
"""Boot with no file against boot with the declared file."""
|
||||
name = entry["name"]
|
||||
evidence = (record_id,)
|
||||
if boot.status == BOOTS:
|
||||
out = [Verdict("boots_without_file", "hle_fallback", True,
|
||||
f"the core reaches the game without {name}: {boot.detail}", evidence, True)]
|
||||
if entry.get("required") is True:
|
||||
out.append(Verdict(
|
||||
"boots_without_file", "required", False,
|
||||
f"the profile says {name} is required, and the core reaches the game "
|
||||
f"without it. The table settles hle_fallback from this run, not required",
|
||||
evidence))
|
||||
return out
|
||||
if boot.status == FAILS:
|
||||
return [
|
||||
Verdict("fails_without_file", "required", True,
|
||||
f"the core reaches the game only with {name}: {boot.detail}", evidence, True),
|
||||
Verdict("fails_without_file", "hle_fallback", False,
|
||||
f"no stand-in for {name}: {boot.detail}", evidence, True),
|
||||
]
|
||||
return [Verdict("inconclusive", "hle_fallback", None,
|
||||
f"the runs with and without {name} settle nothing: {boot.detail}", evidence)]
|
||||
|
||||
|
||||
def rule_path(entry: dict, declared: str, opened: Opened) -> Verdict | None:
|
||||
"""The core opened the file at another place under the system directory."""
|
||||
if opened.dest == declared:
|
||||
return None
|
||||
return Verdict("opened_under_other_path", "path", opened.dest,
|
||||
f"the core opened {entry['name']} as {opened.dest}, the profile declares "
|
||||
f"{declared}", (opened.record_id,), True)
|
||||
|
||||
|
||||
def rule_alias(entry: dict, opened: Opened) -> Verdict | None:
|
||||
"""The core opened the file under a name the entry does not declare."""
|
||||
name = opened.dest.rsplit("/", 1)[-1]
|
||||
known = [entry["name"], *(entry.get("aliases") or ())]
|
||||
if name in known:
|
||||
return None
|
||||
return Verdict("opened_other_name", "aliases", [*(entry.get("aliases") or ()), name],
|
||||
f"the core opened {entry['name']} under the name {name}",
|
||||
(opened.record_id,), True)
|
||||
|
||||
|
||||
def rule_never_opened(entry: dict, record_ids: Sequence[str], content: str) -> Verdict:
|
||||
return Verdict("never_opened", "opened", False,
|
||||
f"{entry['name']} was staged and the core never opened it while running "
|
||||
f"{content}", tuple(record_ids))
|
||||
|
||||
|
||||
def rule_variants(status: str, labels: Iterable[str], entry: dict, record_id: str) -> Verdict | None:
|
||||
"""Copies of one file that change what the core shows: a pack choice to review."""
|
||||
if status != "differs":
|
||||
return None
|
||||
return Verdict("variant_changes_output", "variants", sorted(labels),
|
||||
f"the copies of {entry['name']} change what the core shows",
|
||||
(record_id,))
|
||||
|
||||
|
||||
def merge_evidence(existing: dict | None, verdict: Verdict) -> dict:
|
||||
"""The entry's evidence: map with the verdict's record ids added under its field."""
|
||||
out = {key: list(value) for key, value in (existing or {}).items()}
|
||||
ids = out.setdefault(verdict.field, [])
|
||||
for rid in verdict.evidence:
|
||||
if rid not in ids:
|
||||
ids.append(rid)
|
||||
return {key: out[key] for key in sorted(out)}
|
||||
+27
-8
@@ -269,6 +269,14 @@ def render_printf(fmt: bytes, args: Sequence[int]) -> str:
|
||||
return "".join(out).rstrip("\n")
|
||||
|
||||
|
||||
def uniform_frame(frame: bytes, pixel_bytes: int) -> bool:
|
||||
"""True for a frame of one colour, an empty one included: a screen that
|
||||
shows nothing of the game."""
|
||||
if len(frame) <= pixel_bytes:
|
||||
return True
|
||||
return frame == frame[:pixel_bytes] * (len(frame) // pixel_bytes)
|
||||
|
||||
|
||||
def png_bytes(width: int, height: int, rgb: bytes) -> bytes:
|
||||
def chunk(tag: bytes, body: bytes) -> bytes:
|
||||
return struct.pack(">I", len(body)) + tag + body + struct.pack(">I", zlib.crc32(tag + body))
|
||||
@@ -359,6 +367,7 @@ class Frontend:
|
||||
self.video = hashlib.sha1()
|
||||
self.audio = hashlib.sha1()
|
||||
self.frame_hashes: list[list] = []
|
||||
self.uniform: set[str] = set()
|
||||
self.logs: list[dict] = []
|
||||
self.presses: list[tuple[int, int, int]] = [tuple(p) for p in spec["presses"]]
|
||||
self.unanswered: dict[int, int] = {}
|
||||
@@ -433,7 +442,10 @@ class Frontend:
|
||||
self.video.update(struct.pack("<II", *self.last_size) + hashlib.sha1(self.last_frame).digest())
|
||||
every = self.spec["frame_hash_every"]
|
||||
if every and self.frame % every == 0:
|
||||
self.frame_hashes.append([self.frame, hashlib.sha1(self.last_frame).hexdigest()])
|
||||
digest = hashlib.sha1(self.last_frame).hexdigest()
|
||||
self.frame_hashes.append([self.frame, digest])
|
||||
if digest not in self.uniform and uniform_frame(self.last_frame, PIXEL_BYTES[self.pixel_format]):
|
||||
self.uniform.add(digest)
|
||||
png_dir = self.spec.get("png_dir")
|
||||
if png_dir and self.last_frame:
|
||||
rgb = to_rgb(self.pixel_format, *self.last_size, self.last_frame)
|
||||
@@ -514,6 +526,7 @@ def run_worker(spec: dict) -> dict:
|
||||
observables[f"state:{name}"] = state_vars.get(name)
|
||||
if spec["frame_hash_every"]:
|
||||
observables["frame_hashes"] = front.frame_hashes
|
||||
observables["uniform_frames"] = sorted(front.uniform)
|
||||
core.retro_unload_game()
|
||||
core.retro_deinit()
|
||||
return {
|
||||
@@ -584,7 +597,7 @@ def frontend_identity() -> dict:
|
||||
|
||||
def core_identity(args: argparse.Namespace) -> dict:
|
||||
core = Path(args.core)
|
||||
return {
|
||||
identity = {
|
||||
"file": core.name,
|
||||
"sha256": file_digest(core, "sha256"),
|
||||
"repo": args.core_repo,
|
||||
@@ -592,6 +605,9 @@ def core_identity(args: argparse.Namespace) -> dict:
|
||||
"build_command": args.build_command,
|
||||
"compiler": elf_comment(core) or None,
|
||||
}
|
||||
if args.core_artifact:
|
||||
identity["artifact"] = args.core_artifact
|
||||
return identity
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -769,6 +785,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
common.add_argument("--core-repo", help="source repository URL of the core build")
|
||||
common.add_argument("--core-commit", help="commit the core was built from")
|
||||
common.add_argument("--build-command", help="command and flags that built the core")
|
||||
common.add_argument("--core-artifact", metavar="URL",
|
||||
help="where the .so was downloaded from, such as a buildbot nightly")
|
||||
common.add_argument("--record", metavar="DIR", nargs="?", const=str(ROOT),
|
||||
help="append the record as one line of measurements/<emulator>.jsonl "
|
||||
"in the checkout DIR (default: this one)")
|
||||
@@ -841,7 +859,7 @@ def validate_args(args: argparse.Namespace) -> list[str]:
|
||||
problems.append(f"--vary {dest}={src}: not a file")
|
||||
names = set(BASE_OBSERVABLES) | {f"state:{v}" for v in args.state_var}
|
||||
if args.frame_hash_every:
|
||||
names.add("frame_hashes")
|
||||
names.update(("frame_hashes", "uniform_frames"))
|
||||
if args.observable not in names:
|
||||
problems.append(f"--observable {args.observable}: not one of {', '.join(sorted(names))}")
|
||||
return problems
|
||||
@@ -936,8 +954,9 @@ def default_subject(record: dict) -> str:
|
||||
return f"{record['kind']}-{slugify(content, 30)}-{digest}"
|
||||
|
||||
|
||||
def write_record(directory: Path, full: dict) -> Path:
|
||||
"""Append the record, with its id, to its emulator's .jsonl."""
|
||||
def write_record(directory: Path, full: dict) -> tuple[Path, dict]:
|
||||
"""Append the record, with its id, to its emulator's .jsonl. Returns the
|
||||
file and the record as written."""
|
||||
from measurement_records import line_text, parse_lines, records_path, seal # noqa: PLC0415
|
||||
|
||||
sealed = seal(full)
|
||||
@@ -948,7 +967,7 @@ def write_record(directory: Path, full: dict) -> Path:
|
||||
lines.parent.mkdir(parents=True, exist_ok=True)
|
||||
with lines.open("a", encoding="utf-8") as handle:
|
||||
handle.write(line_text(sealed) + "\n")
|
||||
return lines
|
||||
return lines, sealed
|
||||
|
||||
|
||||
def probe(args: argparse.Namespace) -> int:
|
||||
@@ -999,7 +1018,7 @@ def probe(args: argparse.Namespace) -> int:
|
||||
"repetitions": repetitions,
|
||||
"presses": [{"frame": f, "button": b, "length": n} for f, b, n in args.press],
|
||||
"observables": list(BASE_OBSERVABLES) + [f"state:{v}" for v in args.state_var]
|
||||
+ (["frame_hashes"] if args.frame_hash_every else []),
|
||||
+ (["frame_hashes", "uniform_frames"] if args.frame_hash_every else []),
|
||||
"frame_hash_every": args.frame_hash_every,
|
||||
"joypad_ports": args.joypad_ports,
|
||||
},
|
||||
@@ -1025,7 +1044,7 @@ def probe(args: argparse.Namespace) -> int:
|
||||
record["subject"] = args.subject or default_subject(record)
|
||||
if args.record:
|
||||
try:
|
||||
lines = write_record(Path(args.record), record)
|
||||
lines, record = write_record(Path(args.record), record)
|
||||
except ProbeError as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
@@ -81,6 +81,47 @@ def records_path(emulator: str) -> str:
|
||||
return f"{RECORDS_DIR}/{emulator}.jsonl"
|
||||
|
||||
|
||||
# The campaign's own files beside the records: append-only like them, one
|
||||
# line per finding or per pass, each line hashed to its id. No profile takes
|
||||
# these names (tests/test_campaign.py checks it).
|
||||
FINDINGS = "findings"
|
||||
CAMPAIGN_RUNS = "campaign-runs"
|
||||
CAMPAIGN_FILES = {
|
||||
FINDINGS: ("found_at", "campaign-finding.schema.json"),
|
||||
CAMPAIGN_RUNS: ("started_at", "campaign-run.schema.json"),
|
||||
}
|
||||
|
||||
|
||||
def line_id(line: dict, date_key: str) -> str:
|
||||
body = {key: value for key, value in line.items() if key != "id"}
|
||||
return f"{line[date_key][:10]}-{sha256_hex(canonical(body))[:12]}"
|
||||
|
||||
|
||||
def seal_line(line: dict, date_key: str) -> dict:
|
||||
return {**line, "id": line_id(line, date_key)}
|
||||
|
||||
|
||||
def read_campaign(root: Path, name: str) -> list[dict]:
|
||||
"""The lines of one campaign file, each checked against its id."""
|
||||
path = root / records_path(name)
|
||||
if not path.is_file():
|
||||
return []
|
||||
date_key = CAMPAIGN_FILES[name][0]
|
||||
lines = parse_lines(path.read_text(encoding="utf-8"), records_path(name))
|
||||
problems = []
|
||||
seen: set[str] = set()
|
||||
for number, line in enumerate(lines, 1):
|
||||
rid = str(line.get("id", ""))
|
||||
if not ID_PATTERN.match(rid) or date_key not in line or line_id(line, date_key) != rid:
|
||||
problems.append(f"{records_path(name)}:{number}: id {rid!r} does not hash the line")
|
||||
if rid in seen:
|
||||
problems.append(f"{records_path(name)}:{number}: id {rid} appears twice")
|
||||
seen.add(rid)
|
||||
if problems:
|
||||
raise MeasurementsError("\n ".join(problems))
|
||||
return lines
|
||||
|
||||
|
||||
IDENTITY = ("id", "emulator", "subject", "supersedes", "migrated_from_version")
|
||||
|
||||
|
||||
@@ -177,6 +218,8 @@ def read_records(root: Path) -> list[Stored]:
|
||||
if not path.endswith(".jsonl") or "/" in emulator:
|
||||
problems.append(f"{path}: {RECORDS_DIR}/ holds <emulator>.jsonl files only")
|
||||
continue
|
||||
if emulator in CAMPAIGN_FILES:
|
||||
continue
|
||||
seen: set[str] = set()
|
||||
try:
|
||||
records = parse_lines(files[path].read_text(encoding="utf-8"), path)
|
||||
|
||||
@@ -140,6 +140,80 @@ def build_index(stored: list[mr.Stored], source: dict | None, profiles: dict,
|
||||
}
|
||||
|
||||
|
||||
def campaign_summary(root: Path, stored: list[mr.Stored], source: dict | None) -> dict:
|
||||
"""The last campaign pass and the findings still waiting for review.
|
||||
|
||||
A finding's history is its lines under one key: the latest line is its
|
||||
state, and only an open or flagged one is listed.
|
||||
"""
|
||||
runs = mr.read_campaign(root, mr.CAMPAIGN_RUNS)
|
||||
latest: dict[str, dict] = {}
|
||||
applied = 0
|
||||
for line in mr.read_campaign(root, mr.FINDINGS):
|
||||
latest[line["key"]] = line
|
||||
applied += line["status"] == "applied"
|
||||
where = {s.record["id"]: s for s in stored}
|
||||
links = {}
|
||||
for line in latest.values():
|
||||
for rid in line["evidence"]:
|
||||
found = where.get(rid)
|
||||
if found and source and source.get("commit"):
|
||||
links[rid] = permalink(source["repo"], source["commit"],
|
||||
mr.records_path(found.emulator), found.line)
|
||||
open_items = [{**line, "links": [links.get(rid) for rid in line["evidence"]]}
|
||||
for _, line in sorted(latest.items()) if line["status"] in ("open", "flagged")]
|
||||
return {"last_run": runs[-1] if runs else None, "passes": len(runs), "applied": applied,
|
||||
"open": open_items}
|
||||
|
||||
|
||||
def _value(value: object) -> str:
|
||||
return f"`{json.dumps(value)}`" if value is not None else "nothing"
|
||||
|
||||
|
||||
def campaign_section(summary: dict | None) -> list[str]:
|
||||
"""The campaign's last pass and its open findings, for the explorer page."""
|
||||
if not summary or not summary.get("last_run"):
|
||||
return []
|
||||
last = summary["last_run"]
|
||||
counts = last["counts"]
|
||||
lines = [
|
||||
"## Campaign",
|
||||
"",
|
||||
f"Tests run by themselves every night on an idle computer ([how]({METHOD_PAGE})). "
|
||||
f"Last pass: {last['ended_at'][:10]}, {_text(last['result'])}"
|
||||
+ (f" ({_text(last['detail'])})" if last.get("detail") else "") + ". "
|
||||
f"{counts['done']} tests done, {counts['failed']} failed, {counts['pending']} waiting. "
|
||||
f"{summary['applied']} profile fields were set from a run.",
|
||||
"",
|
||||
]
|
||||
if last["pending_reasons"]:
|
||||
lines += ["| Waiting for | Tests |", "|-------------|------:|"]
|
||||
lines += [f"| {_text(reason)} | {number} |"
|
||||
for reason, number in sorted(last["pending_reasons"].items(), key=lambda kv: (-kv[1], kv[0]))]
|
||||
lines.append("")
|
||||
if last["failed"]:
|
||||
lines += ["Failed tests:", ""]
|
||||
lines += [f"- {_text(item['emulator'])}, `{_text(item['check'])}`: {_text(item['reason'])}"
|
||||
for item in last["failed"]]
|
||||
lines.append("")
|
||||
lines += ["### Open findings", ""]
|
||||
if not summary["open"]:
|
||||
lines += ["None: every finding was settled by a rule or resolved by a later run.", ""]
|
||||
return lines
|
||||
lines += ["A finding is what a run showed that no rule settles. It waits for someone to "
|
||||
"read the emulator's code.", ""]
|
||||
for item in summary["open"]:
|
||||
records = ", ".join(f"[{rid}]({link})" if link else f"`{rid}`"
|
||||
for rid, link in zip(item["evidence"], item["links"]))
|
||||
lines += [
|
||||
f"- **{_text(item['emulator'])}**, `{_text(item['file'])}`, `{_text(item['field'])}` "
|
||||
f"({item['status']}): the profile says {_value(item['profile_says'])}, the run shows "
|
||||
f"{_value(item['run_shows'])}. {_text(item['statement'])}. Records: {records}.",
|
||||
"",
|
||||
]
|
||||
return lines
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# SQLite
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -360,6 +434,7 @@ def build(public: Path, root: Path = ROOT, emulators_dir: Path = ROOT / "emulato
|
||||
source = {"repo": repo, "commit": commit or (records_commit(root) if stored else None)}
|
||||
profiles = load_profiles(emulators_dir)
|
||||
index = build_index(stored, source, profiles, system_names)
|
||||
index["campaign"] = campaign_summary(root, stored, source)
|
||||
paths = [public / INDEX_PATH]
|
||||
(public / INDEX_PATH).parent.mkdir(parents=True, exist_ok=True)
|
||||
write_if_changed(str(public / INDEX_PATH),
|
||||
@@ -485,6 +560,7 @@ def generate_explorer_page(index: dict) -> str:
|
||||
"",
|
||||
"<MeasurementsExplorer />",
|
||||
"",
|
||||
*campaign_section(index.get("campaign")),
|
||||
])
|
||||
|
||||
|
||||
|
||||
@@ -601,6 +601,8 @@ def _measurement_errors(root: Path | None = None) -> list[str]:
|
||||
root = root or ROOT
|
||||
validator = _validator("core-probe-record.schema.json")
|
||||
out: list[str] = []
|
||||
campaign = {mr.records_path(name): _validator(schema)
|
||||
for name, (_, schema) in mr.CAMPAIGN_FILES.items()}
|
||||
for path, file in sorted(mr.tree_files(root).items()):
|
||||
if not path.endswith(".jsonl"):
|
||||
continue
|
||||
@@ -610,10 +612,12 @@ def _measurement_errors(root: Path | None = None) -> list[str]:
|
||||
except json.JSONDecodeError as exc:
|
||||
out.append(f"{path}:{number}: {exc}")
|
||||
continue
|
||||
out.extend(_errors(validator, record, f"{path}:{number}"))
|
||||
out.extend(_errors(campaign.get(path, validator), record, f"{path}:{number}"))
|
||||
if not out:
|
||||
try:
|
||||
mr.read_records(root)
|
||||
for name in mr.CAMPAIGN_FILES:
|
||||
mr.read_campaign(root, name)
|
||||
except mr.MeasurementsError as exc:
|
||||
out.append(str(exc))
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# Test content of the real-condition campaign (scripts/campaign.py), per system.
|
||||
#
|
||||
# Each entry names a catalog entry, never a file: `name` is the exact Redump or
|
||||
# No-Intro name romfetch fetches and verifies (`romfetch get <romfetch> <name>`),
|
||||
# and `reason` says why this game serves the test. Content is read from
|
||||
# .cache/test-content/<romfetch>/<name>/ with its identity.json, and never
|
||||
# enters git. A record keeps only its identity (catalog name, sha1, tracks).
|
||||
#
|
||||
# region the territory of the disc or cartridge: the BIOS entry the
|
||||
# boot test varies is the one whose `region:` holds it, or one
|
||||
# without `region:`
|
||||
# load the file handed to the core: `cue` (the .cue sheet) or a
|
||||
# file name inside the folder; by default the single file
|
||||
# frames frames run per repetition, the same with and without the file
|
||||
# frame_hash_every frames between two sampled frame hashes
|
||||
# pinned never deleted by the campaign, whoever fetched it
|
||||
# boot_file (per system) the declared file the boot test varies, when a
|
||||
# profile declares it under this name or an alias
|
||||
#
|
||||
# The checks a game is used for are the boot test (no file against the
|
||||
# declared file), the files test (every declared file staged, which ones the
|
||||
# core opens) and the benchmark.
|
||||
|
||||
systems:
|
||||
sony-playstation:
|
||||
romfetch: psx
|
||||
# The file the boot test varies, when the profile declares it: the US
|
||||
# BIOS of a US disc, the same file for every core so the runs compare.
|
||||
boot_file: scph5501.bin
|
||||
content:
|
||||
- name: "Bust-A-Move 2 - Arcade Edition (USA)"
|
||||
catalog: redump
|
||||
reason: >-
|
||||
A US disc that goes through the BIOS shell and its licence screen, then
|
||||
reaches the publisher logos and the attract mode within a minute, with
|
||||
no button pressed. Several static screens, so a run with and a run
|
||||
without the BIOS can be compared frame by frame.
|
||||
region: north-america
|
||||
load: cue
|
||||
frames: 3600
|
||||
frame_hash_every: 10
|
||||
pinned: true
|
||||
@@ -0,0 +1,756 @@
|
||||
"""campaign.py runs the real-condition campaign: state, guards, content, rules.
|
||||
|
||||
No network and no real core. The rules and the profile editor are checked on
|
||||
synthetic records and profiles, the guards and the probe retries on a fake
|
||||
host, the content store on folders made in the test. One class builds a
|
||||
libretro core from C, when a compiler is there, and runs whole passes on a
|
||||
synthetic checkout: records, rules applied with their evidence, findings,
|
||||
resume and rerun on a changed core.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
|
||||
import yaml
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT / "scripts"))
|
||||
|
||||
import campaign
|
||||
import campaign_profiles
|
||||
import campaign_rules as rules
|
||||
import measurement_records as mr
|
||||
import sitemeasurements
|
||||
|
||||
TMP = REPO_ROOT / "tmp" / "tests"
|
||||
|
||||
PROFILE = """\
|
||||
emulator: Fake
|
||||
type: libretro
|
||||
cores: [fake]
|
||||
systems: [fake-system]
|
||||
notes: >
|
||||
A profile for the tests.
|
||||
|
||||
files:
|
||||
# -- the BIOS --
|
||||
|
||||
- name: "bios.bin"
|
||||
description: "The BIOS"
|
||||
required: true
|
||||
size: 4
|
||||
validation: [size]
|
||||
aliases:
|
||||
- "Bios.bin"
|
||||
|
||||
- name: "other.bin"
|
||||
required: false
|
||||
"""
|
||||
|
||||
FAKE_CORE = r"""
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
struct retro_system_info { const char *n, *v, *e; bool full, block; };
|
||||
struct retro_game_geometry { unsigned bw, bh, mw, mh; float a; };
|
||||
struct retro_system_timing { double fps, rate; };
|
||||
struct retro_system_av_info { struct retro_game_geometry g; struct retro_system_timing t; };
|
||||
typedef bool (*env_t)(unsigned, void *);
|
||||
typedef void (*video_t)(const void *, unsigned, unsigned, size_t);
|
||||
typedef size_t (*batch_t)(const int16_t *, size_t);
|
||||
typedef int16_t (*input_t)(unsigned, unsigned, unsigned, unsigned);
|
||||
static env_t env; static video_t video; static batch_t batch;
|
||||
static uint16_t pixels[4 * 2]; static int booted; static unsigned frame;
|
||||
void retro_set_environment(env_t e) { bool t = true; env = e; env(18, &t); }
|
||||
void retro_set_video_refresh(video_t v) { video = v; }
|
||||
void retro_set_audio_sample(void *f) { (void)f; }
|
||||
void retro_set_audio_sample_batch(batch_t b) { batch = b; }
|
||||
void retro_set_input_poll(void *f) { (void)f; }
|
||||
void retro_set_input_state(input_t f) { (void)f; }
|
||||
void retro_set_controller_port_device(unsigned port, unsigned device) { (void)port; (void)device; }
|
||||
void retro_get_system_info(struct retro_system_info *i) {
|
||||
i->n = "Fake"; i->v = "1"; i->e = ""; i->full = true; i->block = false; }
|
||||
void retro_get_system_av_info(struct retro_system_av_info *i) {
|
||||
memset(i, 0, sizeof *i); i->t.fps = 60.0; i->t.rate = 44100.0; }
|
||||
void retro_init(void) {}
|
||||
void retro_deinit(void) {}
|
||||
static FILE *try_open(const char *dir, const char *name) {
|
||||
char path[4096]; snprintf(path, sizeof path, "%s/%s", dir, name); return fopen(path, "rb"); }
|
||||
bool retro_load_game(const void *g) {
|
||||
const char *dir = NULL; FILE *f; (void)g; env(9, &dir);
|
||||
f = try_open(dir, "bios.bin");
|
||||
if (!f) f = try_open(dir, "BIOS.BIN");
|
||||
if (f) { fclose(f); booted = 1; } else booted = getenv("FAKE_HLE") != NULL;
|
||||
return true; }
|
||||
void retro_unload_game(void) {}
|
||||
void retro_run(void) {
|
||||
int16_t s[4] = {1, 2, 3, 4};
|
||||
if (getenv("FAKE_HANG")) for (;;) {}
|
||||
for (int i = 0; i < 8; i++) pixels[i] = booted ? (uint16_t)(frame / 2 + i) : 0;
|
||||
frame++;
|
||||
video(pixels, 4, 2, 8); batch(s, 2); }
|
||||
size_t retro_serialize_size(void) { return 0; }
|
||||
bool retro_serialize(void *d, size_t n) { (void)d; (void)n; return false; }
|
||||
"""
|
||||
|
||||
|
||||
def rep(frames: list[str], uniform: list[str] | None = None, loaded: bool = True) -> dict:
|
||||
return {"loaded": loaded, "observables": {
|
||||
"frame_hashes": [[i * 10, h] for i, h in enumerate(frames)],
|
||||
"uniform_frames": sorted(set(uniform or []))}}
|
||||
|
||||
|
||||
def boot_record(absent: list[dict], present: list[dict], rid: str = "2026-10-27-aaaaaaaaaaaa") -> dict:
|
||||
return {"id": rid, "variants": [{"label": "bios.bin-absent", "repetitions": absent},
|
||||
{"label": "bios.bin", "repetitions": present}]}
|
||||
|
||||
|
||||
class BootOutcome(unittest.TestCase):
|
||||
GAME = ("black", "logo", "title", "title2")
|
||||
|
||||
def test_boots_when_screens_are_shared(self) -> None:
|
||||
record = boot_record([rep(["black", "hle", "logo", "title"], ["black"])] * 2,
|
||||
[rep(self.GAME, ["black"])] * 2)
|
||||
boot = rules.boot_outcome(record, "bios.bin-absent", "bios.bin")
|
||||
self.assertEqual((boot.status, boot.shared), (rules.BOOTS, 2))
|
||||
|
||||
def test_one_colour_and_no_load_fail(self) -> None:
|
||||
black = boot_record([rep(["black"], ["black"]), rep([], [], loaded=False)],
|
||||
[rep(self.GAME, ["black"])])
|
||||
self.assertEqual(rules.boot_outcome(black, "bios.bin-absent", "bios.bin").status, rules.FAILS)
|
||||
|
||||
def test_a_shared_blank_screen_proves_nothing(self) -> None:
|
||||
# The black frame both runs start on is one colour: it is no game screen.
|
||||
record = boot_record([rep(["black", "error"], ["black"])], [rep(self.GAME, ["black"])])
|
||||
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
|
||||
rules.INCONCLUSIVE)
|
||||
|
||||
def test_one_shared_screen_is_not_enough(self) -> None:
|
||||
record = boot_record([rep(["logo", "error"])], [rep(self.GAME, ["black"])])
|
||||
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
|
||||
rules.INCONCLUSIVE)
|
||||
|
||||
def test_repetitions_that_disagree_settle_nothing(self) -> None:
|
||||
record = boot_record([rep(["logo", "title"]), rep(["black"], ["black"])],
|
||||
[rep(self.GAME, ["black"])])
|
||||
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
|
||||
rules.INCONCLUSIVE)
|
||||
|
||||
def test_a_reference_that_shows_nothing_settles_nothing(self) -> None:
|
||||
record = boot_record([rep(["logo", "title"])], [rep(["black"], ["black"])])
|
||||
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
|
||||
rules.INCONCLUSIVE)
|
||||
record = boot_record([rep(["logo", "title"])], [rep(self.GAME, loaded=False)])
|
||||
self.assertEqual(rules.boot_outcome(record, "bios.bin-absent", "bios.bin").status,
|
||||
rules.INCONCLUSIVE)
|
||||
|
||||
|
||||
class RuleTable(unittest.TestCase):
|
||||
ENTRY = MappingProxyType({"name": "bios.bin", "required": True, "aliases": ["Bios.bin"]})
|
||||
RID = "2026-10-27-bbbbbbbbbbbb"
|
||||
|
||||
def test_boots_without_settles_hle_and_queues_required(self) -> None:
|
||||
verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "x", 3), self.ENTRY, self.RID)
|
||||
self.assertEqual([(v.field, v.value, v.apply) for v in verdicts],
|
||||
[("hle_fallback", True, True), ("required", False, False)])
|
||||
self.assertEqual(verdicts[0].evidence, (self.RID,))
|
||||
optional = rules.rule_bios_need(rules.Boot(rules.BOOTS, "x", 3),
|
||||
{"name": "a", "required": False}, self.RID)
|
||||
self.assertEqual([v.field for v in optional], ["hle_fallback"])
|
||||
|
||||
def test_fails_without_settles_required_and_no_hle(self) -> None:
|
||||
verdicts = rules.rule_bios_need(rules.Boot(rules.FAILS, "x"), self.ENTRY, self.RID)
|
||||
self.assertEqual([(v.field, v.value, v.apply) for v in verdicts],
|
||||
[("required", True, True), ("hle_fallback", False, True)])
|
||||
|
||||
def test_inconclusive_is_queued(self) -> None:
|
||||
(verdict,) = rules.rule_bios_need(rules.Boot(rules.INCONCLUSIVE, "x"), self.ENTRY, self.RID)
|
||||
self.assertFalse(verdict.apply)
|
||||
|
||||
def test_path_and_alias(self) -> None:
|
||||
self.assertIsNone(rules.rule_path(self.ENTRY, "bios.bin", rules.Opened("bios.bin", self.RID)))
|
||||
moved = rules.rule_path(self.ENTRY, "bios.bin", rules.Opened("sys/bios.bin", self.RID))
|
||||
self.assertEqual((moved.field, moved.value, moved.apply), ("path", "sys/bios.bin", True))
|
||||
self.assertIsNone(rules.rule_alias(self.ENTRY, rules.Opened("Bios.bin", self.RID)))
|
||||
alias = rules.rule_alias(self.ENTRY, rules.Opened("BIOS.BIN", self.RID))
|
||||
self.assertEqual((alias.field, alias.value), ("aliases", ["Bios.bin", "BIOS.BIN"]))
|
||||
|
||||
def test_never_opened_is_flagged_not_applied(self) -> None:
|
||||
verdict = rules.rule_never_opened(self.ENTRY, [self.RID], "Game (USA)")
|
||||
self.assertFalse(verdict.apply)
|
||||
self.assertEqual(verdict.rule, "never_opened")
|
||||
|
||||
def test_variants(self) -> None:
|
||||
self.assertIsNone(rules.rule_variants("same", ["a", "b"], self.ENTRY, self.RID))
|
||||
self.assertFalse(rules.rule_variants("differs", ["b", "a"], self.ENTRY, self.RID).apply)
|
||||
|
||||
def test_agreement_and_evidence(self) -> None:
|
||||
self.assertTrue(rules.agrees({"name": "a"}, rules.Verdict("r", "hle_fallback", False, "", ())))
|
||||
self.assertFalse(rules.agrees({"name": "a"}, rules.Verdict("r", "hle_fallback", True, "", ())))
|
||||
self.assertTrue(rules.agrees(self.ENTRY, rules.Verdict("r", "aliases", ["Bios.bin"], "", ())))
|
||||
merged = rules.merge_evidence({"required": ["x"]}, rules.Verdict("r", "required", True, "", ("y", "x")))
|
||||
self.assertEqual(merged, {"required": ["x", "y"]})
|
||||
|
||||
|
||||
class ProfileEditor(unittest.TestCase):
|
||||
def test_replace_insert_and_keep_the_rest(self) -> None:
|
||||
out = campaign_profiles.set_fields(PROFILE, 0, {
|
||||
"required": False, "hle_fallback": True,
|
||||
"aliases": ["Bios.bin", "BIOS.BIN"], "evidence": {"hle_fallback": ["2026-10-27-cccccccccccc"]}})
|
||||
parsed = yaml.safe_load(out)
|
||||
self.assertEqual(parsed["files"][0]["hle_fallback"], True)
|
||||
self.assertEqual(parsed["files"][0]["aliases"], ["Bios.bin", "BIOS.BIN"])
|
||||
self.assertEqual(parsed["files"][1], {"name": "other.bin", "required": False})
|
||||
self.assertIn(" # -- the BIOS --", out)
|
||||
self.assertIn(' required: false\n hle_fallback: true\n', out)
|
||||
self.assertIn(' - "BIOS.BIN"\n', out)
|
||||
lines = out.splitlines()
|
||||
self.assertEqual(lines[lines.index(' evidence:') + 1], ' hle_fallback: ["2026-10-27-cccccccccccc"]')
|
||||
|
||||
def test_the_last_entry_and_a_no_op(self) -> None:
|
||||
out = campaign_profiles.set_fields(PROFILE, 1, {"path": "sub/other.bin"})
|
||||
self.assertEqual(yaml.safe_load(out)["files"][1]["path"], "sub/other.bin")
|
||||
self.assertEqual(campaign_profiles.set_fields(PROFILE, 1, {"required": False}), PROFILE)
|
||||
|
||||
def test_refusals(self) -> None:
|
||||
with self.assertRaises(campaign_profiles.ProfileEditError):
|
||||
campaign_profiles.set_fields(PROFILE, 2, {"required": False})
|
||||
with self.assertRaises(campaign_profiles.ProfileEditError):
|
||||
campaign_profiles.set_fields("emulator: X\n", 0, {"required": False})
|
||||
with self.assertRaises(campaign_profiles.ProfileEditError):
|
||||
campaign_profiles.set_fields(PROFILE, 0, {"name": "x.bin"})
|
||||
|
||||
def test_every_profile_reads_as_its_entries(self) -> None:
|
||||
# The text layout the editor reads holds in every profile it may edit.
|
||||
for path in sorted((REPO_ROOT / "emulators").glob("*.yml")):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
entries = (yaml.safe_load(text) or {}).get("files") or []
|
||||
if not entries or "\nfiles:\n" not in text:
|
||||
continue
|
||||
with self.subTest(profile=path.name):
|
||||
self.assertEqual(len(campaign_profiles.item_spans(text.splitlines())), len(entries))
|
||||
|
||||
|
||||
class FakeHost(campaign.Host):
|
||||
def __init__(self, loads: list[float] | None = None, free: float = 500.0) -> None:
|
||||
super().__init__()
|
||||
self.loads = list(loads or [0.0])
|
||||
self.now = 0.0
|
||||
self.slept: list[float] = []
|
||||
self.free = free
|
||||
self.calls: list[list[str]] = []
|
||||
self.results: list[object] = []
|
||||
self.loadavg = self._load
|
||||
self.sleep = self._sleep
|
||||
self.clock = lambda: self.now
|
||||
self.free_gb = lambda path: self.free
|
||||
self.cpu_count = 4
|
||||
self.run = self._run
|
||||
|
||||
def _load(self) -> tuple[float, float, float]:
|
||||
value = self.loads.pop(0) if len(self.loads) > 1 else self.loads[0]
|
||||
return (value, value, value)
|
||||
|
||||
def _sleep(self, seconds: float) -> None:
|
||||
self.slept.append(seconds)
|
||||
self.now += seconds
|
||||
|
||||
def _run(self, argv, **kwargs):
|
||||
self.calls.append(list(argv))
|
||||
result = self.results.pop(0) if len(self.results) > 1 else self.results[0]
|
||||
if isinstance(result, BaseException):
|
||||
raise result
|
||||
return result
|
||||
|
||||
|
||||
class Workspace(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
TMP.mkdir(parents=True, exist_ok=True)
|
||||
self.tmp = Path(tempfile.mkdtemp(dir=TMP))
|
||||
self.root = self.tmp / "repo"
|
||||
(self.root / "emulators").mkdir(parents=True)
|
||||
(self.root / "emulators" / "fake.yml").write_text(PROFILE, encoding="utf-8")
|
||||
bios = self.root / "bios" / "Fake" / "bios.bin"
|
||||
bios.parent.mkdir(parents=True)
|
||||
bios.write_bytes(b"BIOS")
|
||||
sha1 = hashlib.sha1(b"BIOS").hexdigest()
|
||||
db = {"files": {sha1: {"path": "bios/Fake/bios.bin", "name": "bios.bin", "size": 4,
|
||||
"sha1": sha1, "md5": hashlib.md5(b"BIOS").hexdigest()}},
|
||||
"indexes": {"by_name": {"bios.bin": [sha1]}, "by_md5": {}, "by_crc32": {},
|
||||
"by_sha256": {}, "by_path_suffix": {}}}
|
||||
(self.root / "database.json").write_text(json.dumps(db), encoding="utf-8")
|
||||
self.content_file = self.tmp / "content.yml"
|
||||
self.content_file.write_text(yaml.safe_dump({"systems": {"fake-system": {
|
||||
"romfetch": "fakesys", "content": [{
|
||||
"name": "Fake Game (World)", "catalog": "no-intro", "reason": "a test",
|
||||
"frames": 12, "frame_hash_every": 2}]}}}), encoding="utf-8")
|
||||
self.game = self.tmp / "content" / "fakesys" / "Fake Game (World)"
|
||||
self.paths = campaign.Paths(root=self.root, home=self.tmp / "home",
|
||||
content=self.tmp / "content", db=self.root / "database.json",
|
||||
content_file=self.content_file)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def put_game(self, folder: Path | None = None, payload: bytes = b"GAME") -> Path:
|
||||
folder = folder or self.game
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
(folder / "game.bin").write_bytes(payload)
|
||||
(folder / "identity.json").write_text(json.dumps({"status": "verified", "files": [
|
||||
{"name": "game.bin", "size": len(payload), "sha1": hashlib.sha1(payload).hexdigest()}]}))
|
||||
return folder
|
||||
|
||||
def make(self, host: campaign.Host | None = None, **settings: object) -> campaign.Campaign:
|
||||
values = {"delays": (1, 2), "echo": False, **settings}
|
||||
return campaign.Campaign(self.paths, campaign.Settings(**values), host or FakeHost(),
|
||||
lock_path=self.tmp / "auto.lock")
|
||||
|
||||
|
||||
class StateAndResume(Workspace):
|
||||
def test_should_run(self) -> None:
|
||||
self.assertTrue(campaign.should_run(None, "a", "d"))
|
||||
done = {"status": "done", "core_sha256": "a", "profile_digest": "d"}
|
||||
self.assertFalse(campaign.should_run(done, "a", "d"))
|
||||
self.assertTrue(campaign.should_run(done, "b", "d"))
|
||||
self.assertTrue(campaign.should_run(done, "a", "e"))
|
||||
self.assertTrue(campaign.should_run({**done, "status": "pending"}, "a", "d"))
|
||||
self.assertFalse(campaign.should_run({**done, "status": "failed"}, "a", "d"))
|
||||
|
||||
def test_evidence_does_not_change_the_digest(self) -> None:
|
||||
profile = yaml.safe_load(PROFILE)
|
||||
before = campaign.profile_digest(profile)
|
||||
profile["files"][0]["evidence"] = {"required": ["2026-10-27-dddddddddddd"]}
|
||||
self.assertEqual(campaign.profile_digest(profile), before)
|
||||
profile["files"][0]["required"] = False
|
||||
self.assertNotEqual(campaign.profile_digest(profile), before)
|
||||
|
||||
def test_state_survives_a_reload(self) -> None:
|
||||
c = self.make()
|
||||
c.pending("fake", "fake/fake-system/x/files", "content: romfetch not found")
|
||||
campaign.save_state(self.paths.state, c.state)
|
||||
again = self.make()
|
||||
self.assertEqual(again.state["checks"]["fake"]["fake/fake-system/x/files"]["status"], "pending")
|
||||
|
||||
def test_budget(self) -> None:
|
||||
self.assertEqual(campaign.parse_budget("6h"), 21600)
|
||||
self.assertEqual(campaign.parse_budget("90m"), 5400)
|
||||
self.assertEqual(campaign.parse_budget("30"), 30)
|
||||
|
||||
|
||||
class Guards(Workspace):
|
||||
def test_busy_host_waits_then_runs(self) -> None:
|
||||
host = FakeHost(loads=[5.0, 5.0, 0.5])
|
||||
c = self.make(host, max_load_wait=600)
|
||||
c.deadline = 10_000
|
||||
c.guard()
|
||||
self.assertEqual(host.slept, [60.0, 60.0])
|
||||
|
||||
def test_busy_host_pauses(self) -> None:
|
||||
host = FakeHost(loads=[5.0])
|
||||
c = self.make(host, max_load_wait=120)
|
||||
c.deadline = 10_000
|
||||
with self.assertRaisesRegex(campaign.Pause, "busy host"):
|
||||
c.guard()
|
||||
|
||||
def test_full_disk_pauses(self) -> None:
|
||||
c = self.make(FakeHost(free=10.0), min_free_gb=30)
|
||||
c.deadline = 10_000
|
||||
with self.assertRaisesRegex(campaign.Pause, "disk"):
|
||||
c.guard()
|
||||
|
||||
def test_budget_pauses(self) -> None:
|
||||
c = self.make()
|
||||
c.deadline = 0
|
||||
with self.assertRaisesRegex(campaign.Pause, "budget"):
|
||||
c.guard()
|
||||
|
||||
def test_the_lock_is_skipped_under_autorun(self) -> None:
|
||||
lock = self.tmp / "held.lock"
|
||||
with campaign.autorun.global_lock(lock):
|
||||
with self.assertRaises(campaign.autorun.LockBusy), campaign.host_lock(lock):
|
||||
pass
|
||||
os.environ[campaign.LOCK_HELD_ENV] = "1"
|
||||
try:
|
||||
with campaign.host_lock(lock):
|
||||
pass
|
||||
finally:
|
||||
del os.environ[campaign.LOCK_HELD_ENV]
|
||||
|
||||
|
||||
class ProbeRuns(Workspace):
|
||||
def completed(self, code: int, stdout: str = "", stderr: str = "") -> subprocess.CompletedProcess:
|
||||
return subprocess.CompletedProcess([], code, stdout, stderr)
|
||||
|
||||
def probe(self, *results: object) -> tuple[campaign.ProbeResult, FakeHost]:
|
||||
host = FakeHost()
|
||||
host.results = list(results)
|
||||
c = self.make(host)
|
||||
c.deadline = 10_000
|
||||
return c.probe(["run", "--core", "x.so"], "fake-run"), host
|
||||
|
||||
def test_timeout_is_failed_and_not_retried(self) -> None:
|
||||
result, host = self.probe(subprocess.TimeoutExpired(["x"], 1))
|
||||
self.assertEqual((result.status, result.reason), ("failed", "timeout"))
|
||||
self.assertEqual(len(host.calls), 1)
|
||||
self.assertIn("killed after", Path(result.log).read_text())
|
||||
result, host = self.probe(self.completed(1, stderr="subprocess.TimeoutExpired: timed out after 900 seconds"))
|
||||
self.assertEqual((result.reason, len(host.calls)), ("timeout", 1))
|
||||
|
||||
def test_a_crashing_core_is_tried_three_times(self) -> None:
|
||||
result, host = self.probe(self.completed(1, stderr="worker exited -11"))
|
||||
self.assertEqual((result.status, result.reason), ("failed", "crashed 3 times"))
|
||||
self.assertEqual(len(host.calls), 3)
|
||||
|
||||
def test_a_crash_then_a_record(self) -> None:
|
||||
record = {"id": "2026-10-27-eeeeeeeeeeee"}
|
||||
result, host = self.probe(self.completed(1, stderr="worker exited -11"),
|
||||
self.completed(0, stdout=json.dumps(record)))
|
||||
self.assertEqual((result.status, result.record, len(host.calls)), ("done", record, 2))
|
||||
|
||||
def test_refusal_on_a_busy_host_pauses(self) -> None:
|
||||
with self.assertRaisesRegex(campaign.Pause, "busy host"):
|
||||
self.probe(self.completed(3, stderr="load average"))
|
||||
|
||||
|
||||
class Content(Workspace):
|
||||
def content(self, name: str = "Fake Game (World)", pinned: bool = False) -> campaign.Content:
|
||||
return campaign.Content("fake-system", "fakesys", name, "no-intro", "r", None, "", 12, 2,
|
||||
pinned, None)
|
||||
|
||||
def test_local_content_is_verified_by_its_identity(self) -> None:
|
||||
self.put_game()
|
||||
c = self.make()
|
||||
self.assertEqual(c.store.get(self.content()), self.game)
|
||||
(self.game / "game.bin").write_bytes(b"EDIT")
|
||||
with self.assertRaisesRegex(campaign.Unavailable, "does not match"):
|
||||
self.make().store.get(self.content())
|
||||
|
||||
def test_without_romfetch_the_check_waits_for_content(self) -> None:
|
||||
c = self.make(romfetch=str(self.tmp / "absent-romfetch"))
|
||||
with self.assertRaisesRegex(campaign.Unavailable, "romfetch not found"):
|
||||
c.store.get(self.content())
|
||||
|
||||
def test_romfetch_is_retried_on_a_source_error(self) -> None:
|
||||
fetcher = self.tmp / "romfetch"
|
||||
fetcher.write_text("#!/bin/sh\n")
|
||||
fetcher.chmod(0o755)
|
||||
host = FakeHost()
|
||||
error = json.dumps({"error": {"code": "network", "message": "m", "hint": "h"}})
|
||||
game = self.game
|
||||
|
||||
def fetched(*args, **kwargs):
|
||||
self.put_game(game)
|
||||
return subprocess.CompletedProcess([], 0, '{"status": "verified"}', "")
|
||||
host.results = [subprocess.CompletedProcess([], 3, error, ""), None]
|
||||
host.run = lambda argv, **kw: (host.calls.append(argv), fetched() if len(host.calls) == 2
|
||||
else host.results[0])[1]
|
||||
c = self.make(host, romfetch=str(fetcher))
|
||||
self.assertEqual(c.store.get(self.content()), self.game)
|
||||
self.assertEqual(host.slept, [1])
|
||||
self.assertEqual(host.calls[0][1:4], ["get", "fakesys", "Fake Game (World)"])
|
||||
self.assertTrue(c.state["content"]["fakesys/Fake Game (World)"]["fetched_by_campaign"])
|
||||
|
||||
def test_romfetch_not_found_names_its_code(self) -> None:
|
||||
fetcher = self.tmp / "romfetch"
|
||||
fetcher.write_text("#!/bin/sh\n")
|
||||
host = FakeHost()
|
||||
host.results = [subprocess.CompletedProcess(
|
||||
[], 1, json.dumps({"error": {"code": "no_entry", "message": "m", "hint": "h"}}), "")]
|
||||
c = self.make(host, romfetch=str(fetcher))
|
||||
with self.assertRaisesRegex(campaign.Unavailable, r"exit 1 \(no_entry\)"):
|
||||
c.store.get(self.content())
|
||||
self.assertEqual(len(host.calls), 1)
|
||||
|
||||
def test_shared_content_stays_until_its_last_user(self) -> None:
|
||||
c = self.make()
|
||||
self.put_game()
|
||||
content = self.content()
|
||||
c.state["content"][content.key] = {"fetched_by_campaign": True, "pinned": False, "size": 4}
|
||||
c.store.hold(content, "alpha")
|
||||
c.store.hold(content, "beta")
|
||||
c.store.release(content, "alpha")
|
||||
self.assertTrue(self.game.is_dir())
|
||||
c.store.release(content, "beta")
|
||||
self.assertFalse(self.game.exists())
|
||||
|
||||
def test_pinned_or_foreign_content_is_never_deleted(self) -> None:
|
||||
c = self.make()
|
||||
self.put_game()
|
||||
content = self.content(pinned=True)
|
||||
c.state["content"][content.key] = {"fetched_by_campaign": True, "pinned": True}
|
||||
c.store.hold(content, "alpha")
|
||||
c.store.release(content, "alpha")
|
||||
self.assertTrue(self.game.is_dir())
|
||||
c.state["content"][content.key] = {"fetched_by_campaign": False, "pinned": False}
|
||||
c.store.hold(content, "alpha")
|
||||
c.store.release(content, "alpha")
|
||||
self.assertTrue(self.game.is_dir())
|
||||
|
||||
def test_least_recently_used_goes_first_past_the_cap(self) -> None:
|
||||
c = self.make(cache_cap_gb=5e-9)
|
||||
for name, used in (("Old", "2026-01-01"), ("New", "2026-06-01"), ("Kept", "2025-01-01")):
|
||||
self.put_game(self.tmp / "content" / "fakesys" / name)
|
||||
c.state["content"][f"fakesys/{name}"] = {"fetched_by_campaign": True, "pinned": name == "Kept",
|
||||
"size": 4, "last_used": used}
|
||||
self.assertEqual(c.store.evict(keep=set()), ["fakesys/Old"])
|
||||
self.assertTrue((self.tmp / "content" / "fakesys" / "New").is_dir())
|
||||
self.assertTrue((self.tmp / "content" / "fakesys" / "Kept").is_dir())
|
||||
|
||||
def test_load_file(self) -> None:
|
||||
self.put_game()
|
||||
self.assertEqual(campaign.load_file(self.game, ""), self.game / "game.bin")
|
||||
with self.assertRaises(campaign.Unavailable):
|
||||
campaign.load_file(self.game, "cue")
|
||||
|
||||
|
||||
class Findings(Workspace):
|
||||
RID = "2026-10-27-ffffffffffff"
|
||||
|
||||
def setUp(self) -> None:
|
||||
super().setUp()
|
||||
self.c = self.make()
|
||||
self.c.current = campaign.Pass(started=campaign.now_iso())
|
||||
self.content = campaign.Content("fake-system", "fakesys", "Fake Game (World)", "no-intro",
|
||||
"r", None, "", 12, 2, False, None)
|
||||
self.core = campaign.Core("fake", self.tmp / "fake_libretro.so", "a" * 64)
|
||||
self.profile = self.root / "emulators" / "fake.yml"
|
||||
|
||||
def lines(self) -> list[dict]:
|
||||
return mr.read_campaign(self.root, mr.FINDINGS)
|
||||
|
||||
def entry(self, index: int = 0) -> dict:
|
||||
return yaml.safe_load(self.profile.read_text())["files"][index]
|
||||
|
||||
def test_a_settled_field_is_written_with_its_evidence(self) -> None:
|
||||
verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
|
||||
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
|
||||
entry = self.entry()
|
||||
self.assertEqual(entry["hle_fallback"], True)
|
||||
self.assertEqual(entry["evidence"], {"hle_fallback": [self.RID]})
|
||||
self.assertEqual(entry["required"], True)
|
||||
self.assertEqual([(line["field"], line["status"]) for line in self.lines()],
|
||||
[("hle_fallback", "applied"), ("required", "open")])
|
||||
self.assertIn("emulators/fake.yml", self.c.current.touched)
|
||||
# The same outcome a second time adds nothing.
|
||||
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
|
||||
self.assertEqual(len(self.lines()), 2)
|
||||
|
||||
def test_an_agreeing_run_adds_evidence_only(self) -> None:
|
||||
verdicts = rules.rule_bios_need(rules.Boot(rules.FAILS, "nothing"), self.entry(), self.RID)
|
||||
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
|
||||
entry = self.entry()
|
||||
self.assertEqual((entry["required"], entry["hle_fallback"]), (True, False))
|
||||
self.assertEqual(entry["evidence"], {"hle_fallback": [self.RID], "required": [self.RID]})
|
||||
self.assertEqual(self.lines(), [])
|
||||
|
||||
def test_a_later_agreement_resolves_the_open_finding(self) -> None:
|
||||
(verdict,) = rules.rule_bios_need(rules.Boot(rules.INCONCLUSIVE, "?"), self.entry(), self.RID)
|
||||
self.c.settle("fake", self.content, self.core, 0, self.entry(), [verdict])
|
||||
self.assertEqual(self.lines()[-1]["status"], "open")
|
||||
later = rules.Verdict("inconclusive", "hle_fallback", False, "agrees now", ("2026-10-28-000000000000",))
|
||||
self.c.settle("fake", self.content, self.core, 0, self.entry(), [later])
|
||||
self.assertEqual([line["status"] for line in self.lines()], ["open", "resolved"])
|
||||
|
||||
def test_never_opened_is_flagged(self) -> None:
|
||||
verdict = rules.rule_never_opened(self.entry(1), [self.RID], "Fake Game (World)")
|
||||
self.c.settle("fake", self.content, self.core, 1, self.entry(1), [verdict])
|
||||
self.assertEqual(self.lines()[0]["status"], "flagged")
|
||||
self.assertNotIn("evidence", self.entry(1))
|
||||
|
||||
def test_finding_lines_validate(self) -> None:
|
||||
verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
|
||||
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
|
||||
self.c.write_run_line(self.c.current)
|
||||
try:
|
||||
from jsonschema import Draft202012Validator
|
||||
except ImportError:
|
||||
self.skipTest("jsonschema is a tooling dependency")
|
||||
for name, (_, schema_name) in mr.CAMPAIGN_FILES.items():
|
||||
schema = json.loads((REPO_ROOT / "schemas" / schema_name).read_text())
|
||||
for line in mr.read_campaign(self.root, name):
|
||||
self.assertEqual(list(Draft202012Validator(schema).iter_errors(line)), [], name)
|
||||
|
||||
def test_the_batch_commit_plan(self) -> None:
|
||||
verdicts = rules.rule_bios_need(rules.Boot(rules.BOOTS, "shown", 4), self.entry(), self.RID)
|
||||
self.c.settle("fake", self.content, self.core, 0, self.entry(), verdicts)
|
||||
self.c.current.touched.add(mr.records_path("fake"))
|
||||
plan = self.c.plan_commits(set())
|
||||
self.assertEqual(plan, [
|
||||
("chore: add campaign measurements", ["measurements/fake.jsonl", "measurements/findings.jsonl"]),
|
||||
("chore: apply tested fields to fake", ["emulators/fake.yml"])])
|
||||
for subject, _ in plan:
|
||||
self.assertEqual(campaign.autorun.check_subject(subject), [], subject)
|
||||
# A file someone had changed before the pass is left to them.
|
||||
self.assertEqual(self.c.plan_commits({"emulators/fake.yml"})[1:], [])
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out):
|
||||
self.assertEqual(self.c.commit_batch(dry_run=True), [])
|
||||
self.assertIn("would commit 'chore: add campaign measurements'", out.getvalue())
|
||||
|
||||
|
||||
class Registry(unittest.TestCase):
|
||||
def test_no_profile_takes_a_campaign_file_name(self) -> None:
|
||||
stems = {p.stem for p in (REPO_ROOT / "emulators").glob("*.yml")}
|
||||
self.assertEqual(stems & set(mr.CAMPAIGN_FILES), set())
|
||||
|
||||
def test_content_list(self) -> None:
|
||||
content = campaign.load_content(campaign.CONTENT_FILE)
|
||||
systems = {s for p in (REPO_ROOT / "emulators").glob("*.yml")
|
||||
for s in (yaml.safe_load(p.read_text()) or {}).get("systems") or ()}
|
||||
self.assertTrue(content)
|
||||
for system, items in content.items():
|
||||
self.assertIn(system, systems)
|
||||
for item in items:
|
||||
self.assertIn(item.catalog, ("redump", "no-intro"))
|
||||
self.assertGreater(len(item.reason), 40)
|
||||
self.assertGreater(item.frames, item.frame_hash_every)
|
||||
|
||||
def test_the_autorun_job(self) -> None:
|
||||
jobs = campaign.autorun.load_jobs()
|
||||
job = jobs["campaign"]
|
||||
self.assertTrue(job.lock and job.worktree and job.schedule)
|
||||
self.assertEqual(job.gate[0].argv, ("python", "scripts/campaign.py", "gate"))
|
||||
self.assertEqual({p for rule in job.commits for p in rule.paths},
|
||||
{"measurements/*.jsonl", "emulators/*.yml"})
|
||||
self.assertGreater(job.schedule, jobs["nightly"].schedule)
|
||||
|
||||
def test_autorun_tells_its_steps_the_lock_is_held(self) -> None:
|
||||
job = campaign.autorun.load_jobs()["campaign"]
|
||||
self.assertEqual(campaign.autorun.job_env(job)[campaign.LOCK_HELD_ENV], "1")
|
||||
|
||||
|
||||
class SitePage(unittest.TestCase):
|
||||
def test_last_pass_and_open_findings(self) -> None:
|
||||
summary = {
|
||||
"last_run": {"ended_at": "2026-10-27T10:00:00+00:00", "result": "complete", "detail": "",
|
||||
"counts": {"done": 12, "failed": 1, "pending": 3},
|
||||
"pending_reasons": {"standalone": 3},
|
||||
"failed": [{"emulator": "fake", "check": "fake/x/y/bench", "reason": "timeout"}]},
|
||||
"passes": 1, "applied": 2,
|
||||
"open": [{"emulator": "fake", "file": "bios.bin", "field": "required", "status": "open",
|
||||
"profile_says": True, "run_shows": False, "statement": "boots without it",
|
||||
"evidence": ["2026-10-27-aaaaaaaaaaaa"], "links": ["https://x/l#L1"]}],
|
||||
}
|
||||
page = "\n".join(sitemeasurements.campaign_section(summary))
|
||||
self.assertIn("12 tests done, 1 failed, 3 waiting", page)
|
||||
self.assertIn("| standalone | 3 |", page)
|
||||
self.assertIn("the profile says `true`, the run shows `false`", page)
|
||||
self.assertIn("[2026-10-27-aaaaaaaaaaaa](https://x/l#L1)", page)
|
||||
self.assertEqual(sitemeasurements.campaign_section({"last_run": None}), [])
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("cc") and shutil.which("strace"), "no C compiler or no strace")
|
||||
class FakeCorePasses(Workspace):
|
||||
"""Whole passes against a core compiled from FAKE_CORE."""
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
TMP.mkdir(parents=True, exist_ok=True)
|
||||
cls.build = Path(tempfile.mkdtemp(dir=TMP))
|
||||
source = cls.build / "fake.c"
|
||||
source.write_text(FAKE_CORE, encoding="utf-8")
|
||||
cls.core = cls.build / "fake_libretro.so"
|
||||
subprocess.run(["cc", "-shared", "-fPIC", "-o", str(cls.core), str(source)], check=True)
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
shutil.rmtree(cls.build, ignore_errors=True)
|
||||
|
||||
def passing(self, env: dict[str, str] | None = None, core: Path | None = None,
|
||||
**settings: object) -> campaign.Campaign:
|
||||
self.put_game()
|
||||
# The core_probe threshold follows the host: a test never waits on the real load.
|
||||
host = campaign.Host(loadavg=lambda: (0.0, 0.0, 0.0), free_gb=lambda path: 500.0,
|
||||
cpu_count=1_000_000)
|
||||
extra = env or {}
|
||||
|
||||
def run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess:
|
||||
check = bool(kwargs.pop("check", False))
|
||||
return subprocess.run(argv, check=check, env={**os.environ, **extra}, **kwargs)
|
||||
host.run = run
|
||||
values = {"bench_warmup": 2, "bench_frames": 4, "check_repeat": 2, "probe_timeout": 30,
|
||||
"delays": (1,), "echo": False, **settings}
|
||||
c = campaign.Campaign(self.paths, campaign.Settings(**values), host,
|
||||
lock_path=self.tmp / "auto.lock")
|
||||
so = core or self.core
|
||||
c.unit_cores = lambda *a, **k: [campaign.Core("fake", so, campaign.file_sha256(so))]
|
||||
return c
|
||||
|
||||
def test_a_stand_in_is_proven_and_written(self) -> None:
|
||||
c = self.passing({"FAKE_HLE": "1"})
|
||||
current = c.run(set(), None)
|
||||
self.assertEqual(current.result, "complete")
|
||||
records = [s.record for s in mr.read_records(self.root)]
|
||||
kinds = sorted(r["subject"].split("-")[0] for r in records)
|
||||
self.assertEqual(kinds, ["alias", "bench", "boot", "files"])
|
||||
boot = next(r for r in records if r["subject"].startswith("boot"))
|
||||
alias = next(r for r in records if r["subject"].startswith("alias"))
|
||||
entry = yaml.safe_load((self.root / "emulators" / "fake.yml").read_text())["files"][0]
|
||||
self.assertEqual(entry["hle_fallback"], True)
|
||||
self.assertEqual(entry["evidence"], {"aliases": [alias["id"]], "hle_fallback": [boot["id"]]})
|
||||
statuses = {(f["field"], f["status"]) for f in mr.read_campaign(self.root, mr.FINDINGS)}
|
||||
self.assertEqual(statuses, {("hle_fallback", "applied"), ("required", "open"),
|
||||
("aliases", "applied")})
|
||||
runs = mr.read_campaign(self.root, mr.CAMPAIGN_RUNS)
|
||||
self.assertEqual(runs[-1]["counts"]["done"], 4)
|
||||
# The game is pinned by nobody but was not fetched by the campaign: kept.
|
||||
self.assertTrue(self.game.is_dir())
|
||||
|
||||
# A second pass on the same core and profile runs nothing.
|
||||
again = self.passing({"FAKE_HLE": "1"})
|
||||
self.assertEqual(again.run(set(), None).records, [])
|
||||
|
||||
# A new build of the core runs everything again.
|
||||
rebuilt = self.build / "fake2_libretro.so"
|
||||
shutil.copy2(self.core, rebuilt)
|
||||
with rebuilt.open("ab") as handle:
|
||||
handle.write(b"\0")
|
||||
third = self.passing({"FAKE_HLE": "1"}, core=rebuilt)
|
||||
# BIOS.BIN is declared now, so no alias run follows the boot test.
|
||||
self.assertEqual(len(third.run(set(), None).records), 3)
|
||||
|
||||
def test_no_stand_in_settles_required_and_finds_an_alias(self) -> None:
|
||||
c = self.passing()
|
||||
c.run(set(), None)
|
||||
entry = yaml.safe_load((self.root / "emulators" / "fake.yml").read_text())["files"][0]
|
||||
self.assertEqual((entry["required"], entry["hle_fallback"]), (True, False))
|
||||
self.assertIn("BIOS.BIN", entry["aliases"])
|
||||
self.assertEqual(sorted(entry["evidence"]), ["aliases", "hle_fallback", "required"])
|
||||
alias = [f for f in mr.read_campaign(self.root, mr.FINDINGS) if f["field"] == "aliases"]
|
||||
self.assertEqual(alias[0]["status"], "applied")
|
||||
|
||||
def test_a_hung_core_is_killed_and_the_pass_moves_on(self) -> None:
|
||||
c = self.passing({"FAKE_HANG": "1"}, probe_timeout=2)
|
||||
c.run(set(), None)
|
||||
checks = c.state["checks"]["fake"]
|
||||
self.assertTrue(checks)
|
||||
self.assertEqual({entry["status"] for entry in checks.values()}, {"failed"})
|
||||
self.assertEqual({entry["reason"] for entry in checks.values()}, {"timeout"})
|
||||
self.assertEqual(mr.read_records(self.root), [])
|
||||
|
||||
def test_missing_content_leaves_the_checks_pending(self) -> None:
|
||||
c = self.passing(romfetch=str(self.tmp / "no-romfetch"))
|
||||
shutil.rmtree(self.game)
|
||||
c.run(set(), None)
|
||||
reasons = {entry["reason"] for entry in c.state["checks"]["fake"].values()}
|
||||
self.assertEqual(len(reasons), 1)
|
||||
self.assertIn("romfetch not found", reasons.pop())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -154,6 +154,12 @@ class ParsingTests(unittest.TestCase):
|
||||
self.assertEqual(described, f"{library.name} sha256:{core_probe.file_digest(library, 'sha256')}")
|
||||
self.assertEqual(core_probe.describe_env_value("2000-01-01 00:00:00"), "2000-01-01 00:00:00")
|
||||
|
||||
def test_uniform_frame(self) -> None:
|
||||
self.assertTrue(core_probe.uniform_frame(b"", 2))
|
||||
self.assertTrue(core_probe.uniform_frame(b"\x01\x02" * 8, 2))
|
||||
self.assertFalse(core_probe.uniform_frame(b"\x01\x02" * 7 + b"\x01\x03", 2))
|
||||
self.assertFalse(core_probe.uniform_frame(b"\x00\x00\x00\x00\x00\x00\x00\x01", 4))
|
||||
|
||||
def test_cue_tracks(self) -> None:
|
||||
with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp" / "tests") as tmp:
|
||||
cue = Path(tmp, "d.cue")
|
||||
@@ -265,8 +271,8 @@ class FakeCoreTests(unittest.TestCase):
|
||||
for p in record_dir.rglob("*") if p.is_file()], ["measurements/fake.jsonl"])
|
||||
line = json.loads((record_dir / "measurements" / "fake.jsonl").read_text())
|
||||
self.assertEqual(measurement_records.check_line(line), [])
|
||||
# The line is the record the run printed, with its id.
|
||||
self.assertEqual({k: v for k, v in line.items() if k != "id"}, record)
|
||||
# The line is the record the run printed, id included.
|
||||
self.assertEqual(line, record)
|
||||
try:
|
||||
from jsonschema import Draft202012Validator
|
||||
except ImportError:
|
||||
|
||||
Reference in new issue
Block a user