diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 36de3e444..0dd7ecbd9 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -30,6 +30,7 @@ on: - "scripts/largefiles.py" - "scripts/launchers.py" - "scripts/manifestselect.py" + - "scripts/measurement_records.py" - "scripts/nativemode.py" - "scripts/packextras.py" - "scripts/packpaths.py" @@ -46,6 +47,7 @@ on: - "scripts/sitediagrams.py" - "scripts/siteguides.py" - "scripts/sitemarkup.py" + - "scripts/sitemeasurements.py" - "scripts/siteexports.py" - "scripts/siterender.py" - "scripts/sitesources.py" @@ -55,6 +57,7 @@ on: - "scripts/slots.py" - "scripts/sourceref.py" - "scripts/split_pack.py" + - "scripts/sync_measurements.py" - "scripts/truth.py" - "scripts/upstream.py" - "scripts/validate_schemas.py" @@ -69,6 +72,7 @@ on: - "database.json" - "release.json" - "derivable.json" + - "measurements-pin.json" # The Starlight project: config, components, styles, lockfile. Its # generated folders are gitignored and never trigger anything. - "website/**" @@ -114,6 +118,18 @@ jobs: - run: npm ci --prefix website + # The core probe records live in Abdess/retrobios-measurements, read + # at the commit measurements-pin.json names and never at its head: a + # record reaches the site when the pin moves. One shallow fetch of that + # commit, with git alone. + - name: Clone the measurements at their pin + run: | + repo=$(python -c "import json; print(json.load(open('measurements-pin.json'))['repo'])") + commit=$(python -c "import json; print(json.load(open('measurements-pin.json'))['commit'])") + git init -q .cache/measurements + git -C .cache/measurements fetch -q --depth 1 "$repo" "$commit" + git -C .cache/measurements checkout -q --detach "$commit" + - name: Validate data contracts run: python scripts/validate_schemas.py @@ -133,7 +149,7 @@ jobs: - name: Generate site run: | - python scripts/generate_site.py + python scripts/generate_site.py --require-measurements python scripts/generate_readme.py --db database.json --platforms-dir platforms # starlight-links-validator fails the build on a dead link or anchor. npm run build --prefix website diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 7c1cb6af8..1e2910632 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -29,6 +29,7 @@ on: - "database.json" - "release.json" - "derivable.json" + - "measurements-pin.json" - "website/**" # Spelled out twice because the workflow parser reads no YAML anchor. A test # holds the two lists equal. @@ -53,6 +54,7 @@ on: - "database.json" - "release.json" - "derivable.json" + - "measurements-pin.json" - "website/**" permissions: @@ -257,11 +259,21 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # The measurement records at their pin, as deploy-site.yml reads them: + # the pages and the schema check read the same lines. + - name: Clone the measurements at their pin + run: | + repo=$(python -c "import json; print(json.load(open('measurements-pin.json'))['repo'])") + commit=$(python -c "import json; print(json.load(open('measurements-pin.json'))['commit'])") + git init -q .cache/measurements + git -C .cache/measurements fetch -q --depth 1 "$repo" "$commit" + git -C .cache/measurements checkout -q --detach "$commit" + # The pages are gitignored, and the tests that read them fail # in CI without it. The suite runs whatever this step returned, so a # generator that stops still leaves every other test reporting. - name: Generate the site - run: python scripts/generate_site.py + run: python scripts/generate_site.py --require-measurements - name: Run test suite if: ${{ !cancelled() }} diff --git a/scripts/generate_site.py b/scripts/generate_site.py index d8e7ff3d6..eaf3dfc57 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -60,6 +60,7 @@ from common import ( yaml_load, ) from artifacts import ArtifactLockBusy, artifact_lock +from artifacts import json_text as _json_text from cross_reference import FileConsumers, held_copy from nativemode import digest_algorithm, reads_file_contents @@ -108,6 +109,8 @@ from siteguides import ( trust_box, ) import release_record +import sitemeasurements +import sync_measurements # The Starlight project, and where its pages go inside it. STARLIGHT_DIR = "website" @@ -905,15 +908,6 @@ def generate_stats(stats: dict) -> str: return json.dumps(stats, indent=2) + "\n" -def _json_text(value: object) -> str: - """Stable scalar representation for CSV and SQLite exports.""" - if value is None: - return "" - if isinstance(value, (dict, list, tuple, bool)): - return json.dumps(value, ensure_ascii=False, sort_keys=True) - return str(value) - - HASH_ALGORITHMS = ("sha1", "sha256", "md5", "crc32") @@ -1748,6 +1742,9 @@ def generate_data_exports( (downloads / "accepted-hashes.csv", "Accepted hashes, one row per value", "text/csv", None), (downloads / "provenance-wanted.csv", "Catalogued dumps the collection lacks", "text/csv", None), (downloads / "retrobios.sqlite", "SQLite snapshot", "application/vnd.sqlite3", None), + *([(downloads / "measurements.sqlite", "Core probe measurements (SQLite)", + "application/vnd.sqlite3", None)] + if (downloads / "measurements.sqlite").is_file() else []), ] catalog_items: list[dict] = [] published: list[dict] = [] @@ -2076,7 +2073,7 @@ def _join_names(names: list[str]) -> str: return names[0] if len(names) == 1 else ", ".join(names[:-1]) + f" and {names[-1]}" -def generate_data_page(exports: list[dict]) -> str: +def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = None) -> str: manifest_schema = json.loads( (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8")) snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8") @@ -2344,6 +2341,8 @@ def generate_data_page(exports: list[dict]) -> str: "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's " "own file does not carry.", "", + *(sitemeasurements.data_page_section(measurements_sqlite) + if measurements_sqlite and measurements_sqlite.is_file() else []), "## Hosting, rate limits and caching", "", "- Static files on GitHub Pages, read with a plain GET. No key, no account.", @@ -2452,6 +2451,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: "cross-reference.md": "Every emulator core each retrogaming platform runs, the systems it emulates and the upstream project it comes from.", "gaps.md": "Current RetroBIOS verification gaps, platform-to-emulator divergences, missing files and documented source limitations.", "exports.md": "Each platform's own BIOS list (System.dat, batocera-systems, es_bios.xml and the rest) with the hashes and files the emulator source code shows, offered to its maintainers.", + sitemeasurements.EXPLORER_PAGE: "Emulator cores run headless in real conditions: each test with its claim, result, method, computer and full record, filterable, and as SQLite.", "provenance.md": "Hash-based comparison of RetroBIOS metadata with No-Intro, Redump and TOSEC catalog snapshots.", "explore.md": "Systems, emulators and installer options to explore, with the figures of the BIOS collection.", "which-pack.md": "Pick your device or setup and get its BIOS files: the folder they go in, one command that installs and checks them, or a ZIP pack to extract.", @@ -3000,7 +3000,7 @@ def search_weight(relative: Path, level: str) -> float | None: _REFERENCE_PAGES = { "gaps.md", "cross-reference.md", "provenance.md", "exports.md", "data.md", - "files/index.md", "wiki/data-model.md", + "files/index.md", "wiki/data-model.md", sitemeasurements.EXPLORER_PAGE, } @@ -8438,6 +8438,8 @@ def _said_as_list(items: list[str]) -> str: # Data directory members and revisions, read once per build. _DATA_MEMBERS: dict[str, dict[str, dict]] = {} _DATA_REVISIONS: dict[str, dict] = {} +# The measurement index items of each emulator, for its Tested block. +_MEASUREMENT_ITEMS: dict[str, list[dict]] = {} # The emulator pages that list each data directory, by ref. _DATA_READERS: dict[str, list[tuple[str, str]]] = {} @@ -9027,6 +9029,8 @@ def emulator_pages( name, emu_name, number, parts, [cards[i] for i in entries], [files[i] for i in entries]) + lines.extend(sitemeasurements.tested_section(name, _MEASUREMENT_ITEMS.get(name, []))) + blocks = _render_structured_blocks(profile) if blocks: lines.extend(["## Profile data", "", *blocks]) @@ -10944,6 +10948,7 @@ def generate_nav( (60, {"Data and API": "data.md"}, "Data"), (64, {"Data model": "wiki/data-model.md"}, "Data"), (68, {"Updated platform files": "exports.md"}, "Data"), + (69, {"Measurements": sitemeasurements.EXPLORER_PAGE}, "Data"), ], "contribute": [(0, "contributing.md")], } @@ -11002,6 +11007,11 @@ def main() -> None: help="directory the pages are written to (default: the project's " "content collection, src/content/docs)", ) + parser.add_argument( + "--require-measurements", action="store_true", + help="fail when the measurements clone does not hold the pinned commit " + "(the deploy build passes it; without it the site lists no measurements)", + ) args = parser.parse_args() if args.docs_dir is None: args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT) @@ -11158,12 +11168,27 @@ def _generate(args: argparse.Namespace) -> None: if clash: raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}") + print("Compiling the measurements at their pin...") + measured = sync_measurements.build( + public, emulators_dir=Path(args.emulators_dir), require=args.require_measurements, + system_names={sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))}, + ) + for path in measured.paths: + _record(path) + _MEASUREMENT_ITEMS.clear() + _MEASUREMENT_ITEMS.update(sitemeasurements.by_emulator(measured.index)) + print(f" {measured.index['records']} records, {measured.index['count']} tests") + write_if_changed(str(docs / sitemeasurements.EXPLORER_PAGE), + sitemeasurements.generate_explorer_page(measured.index) + reference_next()) + print("Generating static API and bulk data exports...") exports = generate_data_exports( public, db, coverages, profiles, stats, gap_report, consumers ) write_release_advertised(Path(args.site_dir) / "src" / "generated") - write_if_changed(str(docs / "data.md"), generate_data_page(exports) + reference_next()) + write_if_changed(str(docs / "data.md"), + generate_data_page(exports, public / sync_measurements.SQLITE_PATH) + + reference_next()) PAGE_DATA["data.md"] = dataset_structured_data(exports, db) export_reports = load_export_reports() @@ -11319,6 +11344,10 @@ def _generate(args: argparse.Namespace) -> None: for src_file in wiki_src.glob("*.md"): shutil.copy2(src_file, wiki_dest / src_file.name) _record(wiki_dest / src_file.name) + method_page = docs / sitemeasurements.METHOD_PAGE + if method_page.is_file(): + write_if_changed(str(method_page), sitemeasurements.with_hosts( + method_page.read_text(encoding="utf-8"), measured.index)) table_page = docs / SETUP_TABLE_PAGE if table_page.is_file(): write_if_changed(str(table_page), with_setup_table( diff --git a/scripts/sitemeasurements.py b/scripts/sitemeasurements.py new file mode 100644 index 000000000..369ac34ed --- /dev/null +++ b/scripts/sitemeasurements.py @@ -0,0 +1,173 @@ +"""The site's measurement pages, from the index sync_measurements.py writes. + +Three places read the same index: the Tested block of each emulator page, +the Measurements explorer page and the host table of the methodology page. +A number is never shown without its method and host beside it. +""" + +from __future__ import annotations + +import re +import sqlite3 +from pathlib import Path + +import sitemarkup as markup +from siterender import _escape_tags + +EXPLORER_PAGE = "measurements.md" +METHOD_PAGE = "wiki/measurements.md" +HOSTS_MARKER = "" + + +def by_emulator(index: dict) -> dict[str, list[dict]]: + out: dict[str, list[dict]] = {} + for item in index.get("items", []): + out.setdefault(item["emulator"], []).append(item) + return out + + +def _text(value: object) -> str: + """Record text as the page shows it. A semicolon joining two clauses + ends a sentence on the site: the record keeps the words as written.""" + text = " ".join(str(value or "").split()) + text = re.sub(r";\s+(\w)", lambda m: ". " + m[1].upper(), text) + return _escape_tags(text) + + +def _short(commit: str | None) -> str: + return f"`{commit[:12]}`" if commit else "unknown" + + +def tested_item(item: dict) -> list[str]: + """One test: what ran, the result, the method and host beside it, the record.""" + title = item["claim"] or f"Benchmark of {item['content'] or 'the core alone'}" + runs = "1 run" if item["runs"] == 1 else f"{item['runs']} runs, the latest shown" + core = item["core"] + build = f"{_text(core['library'])} at {_short(core['commit'])}" if core.get("commit") \ + else _text(core["library"]) + lines = [ + f"- **{_text(title)}**", + "", + f" **Result:** `{item['status']}`. {_text(item['statement'])}.", + "", + ] + if item.get("method"): + lines += [f" **Method:** {_text(item['method'])}", ""] + lines += [ + f" **Ran on:** {_text(item['host'])}, {item['date'][:10]}. Core build: {build}. " + f"{item['variants']} variant{'s' if item['variants'] != 1 else ''}, " + f"{item['repetitions']} repetition{'s' if item['repetitions'] != 1 else ''} each" + + (f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" if item["flags"] else "") + + ".", + "", + f" [Full record]({item['record_url']}), [its data]({item['data_url']}), {runs}.", + "", + ] + return lines + + +def tested_section(name: str, items: list[dict]) -> list[str]: + """The Tested block of an emulator page, empty when nothing was recorded.""" + if not items: + return [] + lines = [ + "## Tested", + "", + f"This emulator was run in real conditions by the [core probe](../{METHOD_PAGE}). " + f"Each test below is the latest of its kind. " + f"[All measurements](../{EXPLORER_PAGE}), " + f"[these records as SQLite](../downloads/measurements/{name}.sqlite).", + "", + ] + for item in sorted(items, key=lambda i: (i["date"], i["subject"]), reverse=True): + lines += tested_item(item) + return lines + + +def hosts_table(index: dict) -> list[str]: + hosts: dict[str, int] = {} + for item in index.get("items", []): + # The load average differs per run: the computer is what comes before it. + computer = item["host"].split(", load ")[0] + hosts[computer] = hosts.get(computer, 0) + 1 + if not hosts: + return ["No result is recorded at the pinned commit yet.", ""] + return [ + "| Computer | Tests |", + "|----------|------:|", + *(f"| {_text(host)} | {count} |" for host, count in sorted(hosts.items())), + "", + ] + + +def with_hosts(page: str, index: dict) -> str: + """The methodology page with the computers the results ran on.""" + return page.replace(HOSTS_MARKER, "\n".join(hosts_table(index)).rstrip("\n")) + + +def generate_explorer_page(index: dict) -> str: + commit = index.get("commit") + repo = index.get("repo") or "" + source = (f"[retrobios-measurements]({repo}/tree/{commit}) at {_short(commit)}" + if commit else "the measurements project, not read in this build") + return "\n".join([ + "# Measurements", + "", + f"Emulators run in real conditions: {index.get('records', 0)} records, " + f"{index.get('count', 0)} tests, read from {source}. " + f"[How they are made]({METHOD_PAGE}).", + "", + " ".join([ + markup.button("SQLite of every record", "downloads/measurements.sqlite", primary=True), + markup.button("Index as JSON", "data/measurements/index.json"), + ]), + "", + "Each row is the latest record of one test, with its method and the computer it " + "ran on. One SQLite file per emulator is linked from its page.", + "", + "", + "", + ]) + + +def sqlite_tables(path: Path) -> list[tuple[str, list[str]]]: + """Each table of a database with its columns, in the order created.""" + connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + try: + names = [row[0] for row in connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")] + return [(name, [row[1] for row in connection.execute(f"PRAGMA table_info({name})")]) + for name in names] + finally: + connection.close() + + +def data_page_section(path: Path) -> list[str]: + """The Data page's description of measurements.sqlite.""" + return [ + "## Measurements", + "", + "[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe " + "record at the pinned commit of [retrobios-measurements]" + "(https://github.com/Abdess/retrobios-measurements), and " + "`downloads/measurements/.sqlite` the records of one emulator, linked " + f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads " + "[`data/measurements/index.json`](data/measurements/index.json). A record line is " + "valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json) " + "and its data file against [`core-probe-data.schema.json`](api/v1/schemas/core-probe-data.schema.json).", + "", + "| Table | Columns |", + "|-------|---------|", + *(f"| `{name}` | {', '.join(f'`{c}`' for c in columns)} |" for name, columns in sqlite_tables(path)), + "", + "`meta` names the measurements commit and the record version. A list or object " + "value is JSON text, an absent one empty. `runs.superseded` is 1 for a record a " + "later one corrects.", + "", + "```sql", + "SELECT r.emulator, c.claim, c.status, r.recorded_at, h.cpu_model", + "FROM runs r JOIN checks c ON c.run_id = r.id JOIN hosts h ON h.id = r.host_id", + "WHERE r.superseded = 0 ORDER BY r.recorded_at;", + "```", + "", + ] diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py index 949f17744..c766af591 100644 --- a/scripts/sitestarlight.py +++ b/scripts/sitestarlight.py @@ -28,6 +28,7 @@ COMPONENTS = { "Tabs": "@astrojs/starlight/components", "TabItem": "@astrojs/starlight/components", "FilePicker": "components/FilePicker.astro", + "MeasurementsExplorer": "components/MeasurementsExplorer.astro", } # Pages of file cards: level 3 headings by the hundred, so the table of diff --git a/tests/test_measurements.py b/tests/test_measurements.py new file mode 100644 index 000000000..9b3d5c41e --- /dev/null +++ b/tests/test_measurements.py @@ -0,0 +1,441 @@ +"""Core probe records in their own repository, and what the site makes of them. + +A synthetic measurements repository is built with git under tmp/tests: +records split into a line and a sidecar, read at a pinned commit, compiled +into the index and the SQLite files. The append-only rule is checked the +way --fetch applies it, and two branches that each add a line merge. +""" + +from __future__ import annotations + +import contextlib +import copy +import io +import json +import os +import re +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import measurement_records as mr # noqa: E402 +import sitemeasurements # noqa: E402 +import sync_measurements as sm # noqa: E402 + +REPO_URL = "https://github.com/Abdess/retrobios-measurements" +SHA1 = "a" * 40 +SHA256 = "b" * 64 + + +def repetition(value: str, frames: int = 3) -> dict: + return { + "loaded": True, "library_name": "Fake", "library_version": "1", "fps_nominal": 60.0, + "frames": frames, "warmup": 0, "wall_s": 0.5, "ms_per_frame": 2.5, "fps": 400.0, + "observables": {"frame_sha1": value, "video_sha1": value, "audio_sha1": SHA1, + "state_sha1": None, "frame_hashes": [[0, value], [150, value]]}, + "logs": [{"frame": 0, "level": "info", "text": "card present " * 20}], + "options_declared": {"fake_option": ["a", "b"]}, "env_unanswered": {"65": 1}, + "last_frame_size": [4, 2], "tracing": "strace", "input_queries": 3, "input_pressed": 1, + "files_opened": [{"path": "card.bin", "sha1_after": SHA1, "sha1_staged": SHA1}], + "files_probed_absent": [], + } + + +def full_record(emulator: str, subject: str, when: str, kind: str = "check", + values: tuple[str, ...] = ("1" * 40, "2" * 40), observable: str = "video_sha1") -> dict: + variants = [{"label": f"v{i}", "system_files": {"card.bin": SHA1}, + "repetitions": [repetition(v), repetition(v)], "flags": []} + for i, v in enumerate(values)] + record = { + "record_version": mr.RECORD_VERSION, "emulator": emulator, "subject": subject, + "supersedes": None, "kind": kind, "recorded_at": when, + "core": {"file": "fake_libretro.so", "sha256": SHA256, "repo": "https://example.org/fake", + "commit": "c" * 40, "build_command": "make", "compiler": ["GCC 13"], + "library_name": "Fake", "library_version": "1"}, + "frontend": {"script": "scripts/core_probe.py", "commit": "d" * 40, "script_modified": False, + "script_sha256": SHA256}, + "host": {"cpu_model": "Test CPU", "cpu_count": 8, "os": "Linux-6.1-x86_64-with-glibc2.39", + "kernel": "6.1", "machine": "x86_64", "python": "3.12.3", + "load_before": [0.5, 0.5, 0.5], "load_after": [0.6, 0.5, 0.5], "max_load": 2.0}, + "inputs": {"content": {"label": "test disc", "sha1": SHA1, "size": 10, + "tracks": [{"file": "t.bin", "sha1": SHA1, "size": 9}]}, + "options": {"fake_option": "a"}, "env": {}, "frames": 3, "warmup": 0, + "repetitions": 2 if kind == "check" else 5, "presses": [], + "observables": ["frame_sha1", "video_sha1", "audio_sha1", "state_sha1", "frame_hashes"], + "frame_hash_every": 150, "joypad_ports": 2}, + "variants": variants, + "flags": [], + } + if kind == "check": + groups: dict[str, list[str]] = {} + for variant, value in zip(variants, values): + key = json.dumps(variant["repetitions"][0]["observables"][observable], sort_keys=True) + groups.setdefault(key, []).append(variant["label"]) + record["check"] = { + "claim": f"The card decides what {emulator} shows.", "method": "Each card, twice.", + "observable": observable, "variable": "card.bin", + "result": {"status": "differs" if len(groups) > 1 else "same", + "statement": f"{observable} takes {len(groups)} values", + "groups": [{"value": json.loads(k), "variants": v} for k, v in groups.items()]}, + } + else: + record["variants"] = record["variants"][:1] + record["variants"][0]["repetitions"] = [repetition(values[0]) for _ in range(5)] + raw = [2.0, 2.5, 3.0, 2.2, 2.4] + stats = {"median": 2.4, "mean": 2.42, "stdev": 0.37, "min": 2.0, "max": 3.0, "raw": raw} + record["timing"] = {"ms_per_frame": stats, + "fps": {**stats, "median": 416.7, "raw": [1000 / v for v in raw]}} + return record + + +def git(cwd: Path, *args: str) -> str: + env = {**os.environ, "GIT_AUTHOR_NAME": "t", "GIT_AUTHOR_EMAIL": "t@t", "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@t", "GIT_CEILING_DIRECTORIES": str(cwd.parent)} + return subprocess.run(["git", *args], cwd=cwd, env=env, capture_output=True, text=True, + check=True).stdout.strip() + + +def add_record(tree: Path, full: dict) -> dict: + slim, sidecar = mr.split(full) + sealed, blob = mr.seal(slim, sidecar) + data = tree / mr.sidecar_path(sealed["emulator"], sealed["id"]) + data.parent.mkdir(parents=True, exist_ok=True) + data.write_bytes(blob) + lines = tree / mr.records_path(sealed["emulator"]) + lines.parent.mkdir(parents=True, exist_ok=True) + with lines.open("a", encoding="utf-8") as handle: + handle.write(mr.line_text(sealed) + "\n") + return sealed + + +def commit_all(tree: Path, message: str) -> str: + git(tree, "add", "--all", ".") + git(tree, "commit", "-q", "-m", message) + return git(tree, "rev-parse", "HEAD") + + +class Workspace(unittest.TestCase): + """A measurements repository with three records and a pin on it.""" + + def setUp(self) -> None: + (REPO_ROOT / "tmp" / "tests").mkdir(parents=True, exist_ok=True) + self.tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp" / "tests")) + self.addCleanup(shutil.rmtree, self.tmp, True) + self.repo = self.tmp / "remote" + self.repo.mkdir() + git(self.repo, "init", "-q", "-b", "main") + (self.repo / ".gitattributes").write_text("records/*.jsonl merge=union\n") + self.first = add_record(self.repo, full_record("alpha", "card", "2026-10-01T10:00:00+00:00")) + self.bench = add_record(self.repo, full_record("alpha", "speed", "2026-10-01T11:00:00+00:00", + kind="bench")) + self.beta = add_record(self.repo, full_record("beta", "card", "2026-10-02T10:00:00+00:00", + values=("1" * 40, "1" * 40))) + self.commit = commit_all(self.repo, "feat: first records") + self.pin = self.tmp / "pin.json" + sm.write_pin({"repo": str(self.repo), "commit": self.commit}, self.pin) + self.emulators = self.tmp / "emulators" + self.emulators.mkdir() + (self.emulators / "alpha.yml").write_text( + 'emulator: "Alpha"\ntype: libretro\nsystems:\n - sega-saturn\nfiles: []\n') + + def built(self, public: Path, clone: Path | None = None) -> sm.Built: + return sm.build(public, clone or self.repo, self.pin, self.emulators, require=True, + scratch=self.tmp / "scratch", system_names={"sega-saturn": "Sega Saturn"}) + + +class RecordSplit(unittest.TestCase): + def test_round_trip(self) -> None: + for kind in ("check", "bench"): + full = full_record("alpha", "card", "2026-10-01T10:00:00+00:00", kind=kind) + slim, sidecar = mr.split(full) + sealed, blob = mr.seal(slim, sidecar) + with self.subTest(kind=kind): + self.assertEqual(mr.join(sealed, mr.read_sidecar(blob)), full) + self.assertEqual(mr.check_line(sealed), []) + self.assertEqual(mr.check_pair(sealed, blob), []) + self.assertNotIn("logs", json.dumps(sealed["variants"])) + self.assertNotIn("frame_hashes", json.dumps(sealed["variants"])) + + def test_bulky_compared_value_moves_to_the_sidecar(self) -> None: + full = full_record("alpha", "card", "2026-10-01T10:00:00+00:00", observable="frame_hashes") + slim, sidecar = mr.split(full) + groups = slim["check"]["result"]["groups"] + self.assertTrue(all("value" not in g and len(g["value_sha1"]) == 40 for g in groups)) + sealed, blob = mr.seal(slim, sidecar) + self.assertEqual(mr.join(sealed, mr.read_sidecar(blob)), full) + + def test_sidecar_bytes_and_id_are_reproducible(self) -> None: + full = full_record("alpha", "card", "2026-10-01T10:00:00+00:00") + one = mr.seal(*mr.split(full)) + two = mr.seal(*mr.split(copy.deepcopy(full))) + self.assertEqual(one, two) + self.assertRegex(one[0]["id"], r"^2026-10-01-[0-9a-f]{12}$") + + def test_tampering_is_named(self) -> None: + sealed, blob = mr.seal(*mr.split(full_record("alpha", "card", "2026-10-01T10:00:00+00:00"))) + self.assertTrue(mr.check_pair(sealed, blob + b"x")) + changed = {**sealed, "subject": "other"} + self.assertTrue(any("does not hash" in p for p in mr.check_line(changed))) + + def test_a_campaign_line_stays_under_20_kb(self) -> None: + """Eleven variants, two repetitions each, long logs: the line is what + a reader opens, the logs are in the sidecar.""" + full = full_record("alpha", "card", "2026-10-01T10:00:00+00:00", + values=tuple(f"{i:040x}" for i in range(11))) + for variant in full["variants"]: + for rep in variant["repetitions"]: + rep["logs"] = rep["logs"] * 200 + rep["observables"]["frame_hashes"] = [[k, SHA1] for k in range(0, 3000, 150)] + sealed, _blob = mr.seal(*mr.split(full)) + self.assertLess(len(mr.line_text(sealed).encode()), 20_000) + + def test_first_layout_record_keeps_its_id(self) -> None: + old = full_record("alpha", "card", "2026-10-01T10:00:00+00:00") + old = mr.first_layout(old) + rid = f"2026-10-01-{mr.sha256_hex(mr.canonical(old))[:12]}" + full = {**old, "record_version": 2, "emulator": "alpha", "subject": "card", + "supersedes": None, "migrated_from_version": 1} + sealed, blob = mr.seal(*mr.split(full), rid) + self.assertEqual(sealed["id"], rid) + self.assertEqual(mr.check_line(sealed), []) + self.assertEqual(mr.check_pair(sealed, blob), []) + self.assertEqual(mr.first_layout(mr.join(sealed, mr.read_sidecar(blob))), old) + forged = {**sealed, "id": "2026-10-01-000000000000"} + self.assertTrue(mr.check_pair(forged, blob)) + + +class SyncAndIndex(Workspace): + def test_records_are_read_at_the_pin_not_the_working_tree(self) -> None: + add_record(self.repo, full_record("alpha", "card", "2026-10-03T10:00:00+00:00")) + records = sm.read_records(self.repo, self.commit) + self.assertEqual(len(records), 3) + + def test_index_shape(self) -> None: + index = self.built(self.tmp / "public").index + self.assertEqual({k: index[k] for k in ("schema_version", "kind", "repo", "commit", "records", "count")}, + {"schema_version": 1, "kind": "measurements", "repo": str(self.repo), + "commit": self.commit, "records": 3, "count": 3}) + keys = {"emulator", "emulator_name", "profiled", "systems", "system_names", "subject", "kind", + "claim", "method", "observable", "variable", "status", "statement", "content", + "variants", "repetitions", "flags", "date", "host", "core", "frontend_commit", "id", + "runs", "record_url", "data_url", "sqlite"} + for item in index["items"]: + self.assertEqual(set(item), keys) + alpha = {i["subject"]: i for i in index["items"] if i["emulator"] == "alpha"} + self.assertEqual(alpha["card"]["status"], "differs") + self.assertEqual(alpha["card"]["system_names"], ["Sega Saturn"]) + self.assertEqual(alpha["card"]["record_url"], + f"{self.repo}/blob/{self.commit}/records/alpha.jsonl#L1") + self.assertEqual(alpha["speed"]["status"], "measured") + self.assertIn("ms per frame", alpha["speed"]["statement"]) + self.assertIn("Test CPU", alpha["speed"]["host"]) + written = json.loads((self.tmp / "public" / sm.INDEX_PATH).read_text()) + self.assertEqual(written, index) + + def test_latest_record_of_a_subject_wins_and_a_superseded_one_is_dropped(self) -> None: + later = full_record("alpha", "card", "2026-10-05T10:00:00+00:00", values=("3" * 40, "3" * 40)) + later["supersedes"] = self.first["id"] + add_record(self.repo, later) + sm.write_pin({"repo": str(self.repo), "commit": commit_all(self.repo, "feat: rerun")}, self.pin) + card = next(i for i in self.built(self.tmp / "public").index["items"] + if i["emulator"] == "alpha" and i["subject"] == "card") + self.assertEqual((card["status"], card["runs"]), ("same", 1)) + self.assertTrue(card["record_url"].endswith("#L3")) + + def test_sqlite_tables_and_row_counts_match_the_records(self) -> None: + public = self.tmp / "public" + self.built(public) + connection = sqlite3.connect(public / sm.SQLITE_PATH) + tables = [row[0] for row in connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")] + self.assertEqual(tuple(tables), sm.TABLES) + count = lambda table: connection.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] # noqa: E731 + records = [s.record for s in sm.read_records(self.repo, self.commit)] + self.assertEqual(count("runs"), len(records)) + self.assertEqual(count("checks"), sum(1 for r in records if r.get("check"))) + self.assertEqual(count("benchmarks"), 2 * sum(1 for r in records if r.get("timing"))) + self.assertEqual(count("variants"), sum(len(r["variants"]) for r in records)) + self.assertEqual(count("check_groups"), + sum(len(r["check"]["result"]["groups"]) for r in records if r.get("check"))) + self.assertEqual(count("hosts"), 1) + self.assertEqual(count("cores"), 1) + self.assertEqual(dict(connection.execute("SELECT key, value FROM meta"))["measurements_commit"], + self.commit) + self.assertEqual(connection.execute("SELECT system FROM emulator_systems").fetchall(), + [("sega-saturn",)]) + indexes = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type = 'index'")} + self.assertTrue({"runs_emulator_idx", "emulator_systems_system_idx", "checks_claim_idx"} <= indexes) + connection.close() + for emulator, expected in (("alpha", 2), ("beta", 1)): + per = sqlite3.connect(public / sm.PER_EMULATOR_DIR / f"{emulator}.sqlite") + self.assertEqual(per.execute("SELECT COUNT(*), MIN(emulator), MAX(emulator) FROM runs").fetchone(), + (expected, emulator, emulator)) + per.close() + + def test_two_builds_are_byte_identical(self) -> None: + one, two = self.tmp / "one", self.tmp / "two" + first, second = self.built(one), self.built(two) + self.assertEqual([p.relative_to(one) for p in first.paths], [p.relative_to(two) for p in second.paths]) + for path in first.paths: + self.assertEqual(path.read_bytes(), (two / path.relative_to(one)).read_bytes(), path.name) + + def test_a_missing_pin_fails_only_when_required(self) -> None: + sm.write_pin({"repo": str(self.repo), "commit": "f" * 40}, self.pin) + with self.assertRaises(sm.MeasurementsError): + self.built(self.tmp / "public") + with contextlib.redirect_stderr(io.StringIO()) as err: + built = sm.build(self.tmp / "public", self.repo, self.pin, self.emulators, require=False, + scratch=self.tmp / "scratch") + self.assertIn("WARNING", err.getvalue()) + self.assertEqual((built.index["count"], built.index["commit"]), (0, None)) + + def test_every_line_validates_against_the_schema(self) -> None: + try: + from jsonschema import Draft202012Validator + except ImportError: + self.skipTest("jsonschema is not installed") + import validate_schemas # noqa: PLC0415 + + self.assertEqual(validate_schemas._measurement_errors(self.repo, self.pin), []) + lines = self.repo / mr.records_path("beta") + record = json.loads(lines.read_text()) + record["kind"] = "guess" + lines.write_text(mr.line_text(record) + "\n") + sm.write_pin({"repo": str(self.repo), "commit": commit_all(self.repo, "bad")}, self.pin) + errors = validate_schemas._measurement_errors(self.repo, self.pin) + self.assertTrue(any("records/beta.jsonl:1" in e and "kind" in e for e in errors), errors) + self.assertIsNotNone(Draft202012Validator) + + +class AppendOnly(Workspace): + def fetch(self, *extra: str) -> tuple[int, str]: + err = io.StringIO() + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(err): + code = sm.main(["--fetch", "--clone", str(self.tmp / "clone"), "--pin", str(self.pin), *extra]) + return code, err.getvalue() + + def test_an_appended_line_moves_the_pin(self) -> None: + self.assertEqual(self.fetch()[0], 0) + add_record(self.repo, full_record("alpha", "card", "2026-10-04T10:00:00+00:00")) + head = commit_all(self.repo, "feat: rerun") + self.assertEqual(self.fetch("--bump")[0], 0) + self.assertEqual(sm.load_pin(self.pin)["commit"], head) + + def test_a_changed_line_is_refused(self) -> None: + lines = self.repo / mr.records_path("alpha") + text = lines.read_text().splitlines() + text[0] = text[0].replace('"Each card, twice."', '"Each card, once."') + lines.write_text("\n".join(text) + "\n") + commit_all(self.repo, "rewrite") + code, err = self.fetch("--bump") + self.assertEqual(code, 1) + self.assertIn("records/alpha.jsonl: line 1 changed", err) + self.assertEqual(sm.load_pin(self.pin)["commit"], self.commit) + + def test_a_removed_line_or_data_file_is_refused(self) -> None: + lines = self.repo / mr.records_path("alpha") + lines.write_text(lines.read_text().splitlines()[0] + "\n") + (self.repo / mr.sidecar_path("beta", self.beta["id"])).unlink() + commit_all(self.repo, "remove") + code, err = self.fetch() + self.assertEqual(code, 1) + self.assertIn("records/alpha.jsonl: line 2 removed", err) + self.assertIn(f"{mr.sidecar_path('beta', self.beta['id'])}: removed", err) + + def test_two_branches_adding_a_line_merge_cleanly(self) -> None: + git(self.repo, "checkout", "-q", "-b", "left") + add_record(self.repo, full_record("alpha", "card", "2026-10-06T10:00:00+00:00")) + commit_all(self.repo, "left") + git(self.repo, "checkout", "-q", "main") + git(self.repo, "checkout", "-q", "-b", "right") + add_record(self.repo, full_record("alpha", "speed", "2026-10-06T11:00:00+00:00", kind="bench", + values=("4" * 40,))) + commit_all(self.repo, "right") + git(self.repo, "merge", "-q", "--no-edit", "left") + merged = git(self.repo, "rev-parse", "HEAD") + self.assertEqual(sm.append_only_problems(self.repo, self.commit, merged), []) + records = sm.read_records(self.repo, merged) + self.assertEqual(sum(1 for s in records if s.emulator == "alpha"), 4) + + +class SitePages(unittest.TestCase): + ITEM = { + "emulator": "beetle_saturn", "emulator_name": "Beetle Saturn", "profiled": True, + "systems": ["sega-saturn"], "system_names": ["Sega Saturn"], "subject": "mpeg-card", + "kind": "check", "claim": "The card decides it.", "method": "Twice per card; every input fixed.", + "observable": "video_sha1", "variable": "mpeg_card.bin", "status": "same", + "statement": "video_sha1 is identical across all 11 variants", "content": "disc", + "variants": 11, "repetitions": 2, "flags": 1, "date": "2026-10-27T05:14:32+00:00", + "host": "Test CPU, 12 CPUs", "core": {"library": "Beetle Saturn v1.32.1", "commit": "2ce77e2" * 5 + "abcde", + "file": "x.so", "sha256": SHA256, "repo": None, "build": "make"}, + "frontend_commit": None, "id": "2026-10-27-43527b763ace", "runs": 2, + "record_url": f"{REPO_URL}/blob/{'e' * 40}/records/beetle_saturn.jsonl#L4", + "data_url": f"{REPO_URL}/blob/{'e' * 40}/data/beetle_saturn/2026-10-27-43527b763ace.json.gz", + "sqlite": "downloads/measurements/beetle_saturn.sqlite", + } + + def test_tested_block(self) -> None: + self.assertEqual(sitemeasurements.tested_section("beetle_saturn", []), []) + text = "\n".join(sitemeasurements.tested_section("beetle_saturn", [self.ITEM])) + self.assertTrue(text.startswith("## Tested\n")) + for expected in ("**Result:** `same`", "**Method:** Twice per card. Every input fixed.", "**Ran on:** Test CPU, 12 CPUs", + self.ITEM["record_url"], self.ITEM["data_url"], "1 flag raised", "2 runs", + "../downloads/measurements/beetle_saturn.sqlite", "../wiki/measurements.md", + "<image>"): + self.assertIn(expected, text) + + def test_hosts_table_on_the_method_page(self) -> None: + page = sitemeasurements.with_hosts(f"x\n\n{sitemeasurements.HOSTS_MARKER}\n", + {"items": [self.ITEM, {**self.ITEM, "host": "Test CPU, 12 CPUs, load 1 before, 2 after"}]}) + self.assertIn("| Test CPU, 12 CPUs | 2 |", page) + self.assertNotIn(sitemeasurements.HOSTS_MARKER, page) + + def test_the_explorer_reads_the_index(self) -> None: + source = (REPO_ROOT / "website" / "src" / "components" / "MeasurementsExplorer.astro").read_text() + self.assertIn("data/measurements/index.json", source) + self.assertEqual(sm.INDEX_PATH.as_posix(), "data/measurements/index.json") + read = set(re.findall(r"\b(?:item|i)\.([a-z_]+)", source)) - {"length"} + self.assertTrue(read) + self.assertLessEqual(read, set(self.ITEM), read - set(self.ITEM)) + page = sitemeasurements.generate_explorer_page({"records": 7, "count": 7, "commit": "e" * 40, + "repo": REPO_URL}) + self.assertIn("", page) + self.assertIn("downloads/measurements.sqlite", page) + + def test_the_deploy_clones_the_measurements_at_the_pin(self) -> None: + workflow = (REPO_ROOT / ".github" / "workflows" / "deploy-site.yml").read_text() + clone = workflow.index("Clone the measurements at their pin") + step = workflow[clone:workflow.index("- name:", clone)] + self.assertIn("measurements-pin.json", step) + self.assertIn('fetch -q --depth 1 "$repo" "$commit"', step) + self.assertIn('checkout -q --detach "$commit"', step) + self.assertNotIn("uses:", step) + self.assertLess(clone, workflow.index("Validate data contracts")) + self.assertLess(clone, workflow.index("python scripts/generate_site.py --require-measurements")) + self.assertIn('- "measurements-pin.json"', workflow) + pin = sm.load_pin() + self.assertEqual(pin["repo"], REPO_URL) + + +class Freshness(unittest.TestCase): + def test_pin_against_the_remote_head(self) -> None: + import check_freshness as cf # noqa: PLC0415 + + pin = sm.load_pin() + self.assertEqual(cf.check_measurements(False, head=lambda _repo: pin["commit"]).status, cf.OK) + stale = cf.check_measurements(False, head=lambda _repo: "f" * 40) + self.assertEqual(stale.status, cf.STALE) + self.assertIn("sync_measurements.py --fetch --bump", stale.detail) + self.assertEqual(cf.check_measurements(True).status, cf.SKIPPED) + self.assertIn("measurements", cf.AREAS) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_copy.py b/tests/test_site_copy.py index ebf809516..ac3991abd 100644 --- a/tests/test_site_copy.py +++ b/tests/test_site_copy.py @@ -36,6 +36,10 @@ START_PAGES = { "wiki/faq.md": 450, "wiki/glossary.md": 200, } +# Reference pages written for anyone, held to the Start here budgets. +START_LEVEL_PAGES = { + "wiki/measurements.md": 300, +} # Level 1: the guide part of each setup page, above its Reference heading. SETUP_CEILING = 220 # The setup index is a card per setup, one sentence each. @@ -321,7 +325,8 @@ def measure(page: Page) -> dict: def held_pages(docs: Path) -> list[tuple[str, str, int]]: """(relative path, level, ceiling) for every page the budgets hold.""" - pages = [(name, "start", ceiling) for name, ceiling in START_PAGES.items()] + pages = [(name, "start", ceiling) + for name, ceiling in {**START_PAGES, **START_LEVEL_PAGES}.items()] pages += [ (sitepages.name(path), "setup", SETUP_CEILING) for path in sorted((docs / "platforms").glob("*.md*")) diff --git a/tests/test_site_r4_reference.py b/tests/test_site_r4_reference.py index 6100cae49..874445320 100644 --- a/tests/test_site_r4_reference.py +++ b/tests/test_site_r4_reference.py @@ -248,6 +248,13 @@ class SourceLinksOnlyWhereTheFileIs(unittest.TestCase): def test_every_rendered_link_is_recorded(self): record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) pages = sorted(_built(DOCS / "emulators").rglob("*.md*")) + sorted(DOCS.glob("gaps*.md*")) + import sync_measurements # noqa: PLC0415 + + # A Tested block links the measurements repository at its pin. The + # record of source trees does not cover it: those paths are checked + # against the pinned tree of the clone itself. + measured_slug = urllib.parse.urlsplit(sync_measurements.load_pin()["repo"]).path.strip("/") + measured: dict[str, set[str]] = {} checked, stray = 0, [] for page in pages: for found in _PERMALINK.finditer(page.read_text(encoding="utf-8")): @@ -256,7 +263,17 @@ class SourceLinksOnlyWhereTheFileIs(unittest.TestCase): key = f"{found['host']}/{found['slug']}@{found['pin']}" path = urllib.parse.unquote(found["path"]) checked += 1 - if path not in record.get(key, ()): + if found["slug"] == measured_slug: + if found["pin"] not in measured: + if not sync_measurements.has_commit(sync_measurements.CLONE, found["pin"]): + stray.append((page.name, key, "pin not in the measurements clone")) + measured[found["pin"]] = set() + continue + measured[found["pin"]] = set( + sync_measurements.tree_files(sync_measurements.CLONE, found["pin"])) + if path not in measured[found["pin"]]: + stray.append((page.name, key, path)) + elif path not in record.get(key, ()): stray.append((page.name, key, path)) self.assertGreater(checked, 1000) self.assertEqual(stray[:20], []) diff --git a/website/src/components/MeasurementsExplorer.astro b/website/src/components/MeasurementsExplorer.astro new file mode 100644 index 000000000..dd4d7dd45 --- /dev/null +++ b/website/src/components/MeasurementsExplorer.astro @@ -0,0 +1,169 @@ +--- +// The Measurements page: every test of the measurements index, filtered in +// the browser. Reads public/data/measurements/index.json, which +// scripts/sync_measurements.py writes from the records at the pinned commit. +// Each row keeps its method and the computer it ran on beside the result, +// and links its full record. The filters answer from the index alone: no +// SQL engine is loaded. +const base = import.meta.env.BASE_URL.replace(/\/?$/, '/'); +--- + +
+
+ + + + + +
+

Loading the measurements...

+
+ + + + + + + + + + + + +
DateEmulatorTest and methodResultRan onRecord
+
+
+ + + + diff --git a/wiki/measurements.md b/wiki/measurements.md new file mode 100644 index 000000000..88d5daf85 --- /dev/null +++ b/wiki/measurements.md @@ -0,0 +1,47 @@ +--- +level: reference +order: 70 +group: Data +terms: measurement, benchmark, tested, core probe, real conditions, method +--- + +# How emulators are tested + +Some pages say what an emulator does with a file. Where we ran it to check, +the emulator's page shows a **Tested** block. Every result is on the +[Measurements](../measurements.md) page. + +## What a test does + +- A script runs the emulator with no window and no sound. +- It starts a game or a disc and presses buttons at set frames. +- It runs again for each version of one file, such as a BIOS. Nothing else changes. +- It compares what the emulator showed, played or saved, by fingerprint (a hash). +- Each version runs at least twice, each time from a fresh start. + +## What a result says + +- **same**: the file changed nothing that was compared. +- **differs**: some versions of the file changed it. +- **inconclusive**: two runs of one version did not match, so no answer is given. +- **measured**: a speed test, given as time per frame. + +## What a result keeps + +Each result names its method, the computer it ran on and the exact build of +the emulator. Its inputs are kept by fingerprint. A result holds for that +build and those inputs only. + +A speed figure depends on the computer and how busy it was. It is never +shown without them. + +## Where results are kept + +Each result is a full record in the +[retrobios-measurements](https://github.com/Abdess/retrobios-measurements) +project. A record is never changed once added. A new run is added beside it, +and a correction names the record it replaces. + +The results so far ran on these computers: + +