feat: show measurements on the site

Each emulator with records gets a Tested block: claim, result, method, computer, core build and the permalink of its record at the pin. A Measurements page filters every test in the browser from the compact index, and the Data page lists measurements.sqlite with its tables. One SQLite file per emulator is linked from its page. wiki/measurements.md says how a test runs, held to the Start here budgets.

Both workflows fetch the pinned commit alone with git before the build, and the deploy fails without it.

scripts/generate_site.py

@@ -60,6 +60,7 @@ from common import (
     yaml_load,
 )
 from artifacts import ArtifactLockBusy, artifact_lock
+from artifacts import json_text as _json_text
 from cross_reference import FileConsumers, held_copy
 from nativemode import digest_algorithm, reads_file_contents
 
@@ -108,6 +109,8 @@ from siteguides import (
     trust_box,
 )
 import release_record
+import sitemeasurements
+import sync_measurements
 
 # The Starlight project, and where its pages go inside it.
 STARLIGHT_DIR = "website"
@@ -905,15 +908,6 @@ def generate_stats(stats: dict) -> str:
     return json.dumps(stats, indent=2) + "\n"
 
 
-def _json_text(value: object) -> str:
-    """Stable scalar representation for CSV and SQLite exports."""
-    if value is None:
-        return ""
-    if isinstance(value, (dict, list, tuple, bool)):
-        return json.dumps(value, ensure_ascii=False, sort_keys=True)
-    return str(value)
-
-
 HASH_ALGORITHMS = ("sha1", "sha256", "md5", "crc32")
 
 
@@ -1748,6 +1742,9 @@ def generate_data_exports(
         (downloads / "accepted-hashes.csv", "Accepted hashes, one row per value", "text/csv", None),
         (downloads / "provenance-wanted.csv", "Catalogued dumps the collection lacks", "text/csv", None),
         (downloads / "retrobios.sqlite", "SQLite snapshot", "application/vnd.sqlite3", None),
+        *([(downloads / "measurements.sqlite", "Core probe measurements (SQLite)",
+            "application/vnd.sqlite3", None)]
+          if (downloads / "measurements.sqlite").is_file() else []),
     ]
     catalog_items: list[dict] = []
     published: list[dict] = []
@@ -2076,7 +2073,7 @@ def _join_names(names: list[str]) -> str:
     return names[0] if len(names) == 1 else ", ".join(names[:-1]) + f" and {names[-1]}"
 
 
-def generate_data_page(exports: list[dict]) -> str:
+def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = None) -> str:
     manifest_schema = json.loads(
         (_REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text(encoding="utf-8"))
     snippet = (_REPO_ROOT / "scripts" / "check_platform_folder.py").read_text(encoding="utf-8")
@@ -2344,6 +2341,8 @@ def generate_data_page(exports: list[dict]) -> str:
         "- `curated`, `shared_group` and `mirror_of` mark an entry the platform's "
         "own file does not carry.",
         "",
+        *(sitemeasurements.data_page_section(measurements_sqlite)
+          if measurements_sqlite and measurements_sqlite.is_file() else []),
         "## Hosting, rate limits and caching",
         "",
         "- Static files on GitHub Pages, read with a plain GET. No key, no account.",
@@ -2452,6 +2451,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str:
         "cross-reference.md": "Every emulator core each retrogaming platform runs, the systems it emulates and the upstream project it comes from.",
         "gaps.md": "Current RetroBIOS verification gaps, platform-to-emulator divergences, missing files and documented source limitations.",
         "exports.md": "Each platform's own BIOS list (System.dat, batocera-systems, es_bios.xml and the rest) with the hashes and files the emulator source code shows, offered to its maintainers.",
+        sitemeasurements.EXPLORER_PAGE: "Emulator cores run headless in real conditions: each test with its claim, result, method, computer and full record, filterable, and as SQLite.",
         "provenance.md": "Hash-based comparison of RetroBIOS metadata with No-Intro, Redump and TOSEC catalog snapshots.",
         "explore.md": "Systems, emulators and installer options to explore, with the figures of the BIOS collection.",
         "which-pack.md": "Pick your device or setup and get its BIOS files: the folder they go in, one command that installs and checks them, or a ZIP pack to extract.",
@@ -3000,7 +3000,7 @@ def search_weight(relative: Path, level: str) -> float | None:
 
 _REFERENCE_PAGES = {
     "gaps.md", "cross-reference.md", "provenance.md", "exports.md", "data.md",
-    "files/index.md", "wiki/data-model.md",
+    "files/index.md", "wiki/data-model.md", sitemeasurements.EXPLORER_PAGE,
 }
 
 
@@ -8438,6 +8438,8 @@ def _said_as_list(items: list[str]) -> str:
 # Data directory members and revisions, read once per build.
 _DATA_MEMBERS: dict[str, dict[str, dict]] = {}
 _DATA_REVISIONS: dict[str, dict] = {}
+# The measurement index items of each emulator, for its Tested block.
+_MEASUREMENT_ITEMS: dict[str, list[dict]] = {}
 # The emulator pages that list each data directory, by ref.
 _DATA_READERS: dict[str, list[tuple[str, str]]] = {}
 
@@ -9027,6 +9029,8 @@ def emulator_pages(
                     name, emu_name, number, parts, [cards[i] for i in entries],
                     [files[i] for i in entries])
 
+    lines.extend(sitemeasurements.tested_section(name, _MEASUREMENT_ITEMS.get(name, [])))
+
     blocks = _render_structured_blocks(profile)
     if blocks:
         lines.extend(["## Profile data", "", *blocks])
@@ -10944,6 +10948,7 @@ def generate_nav(
             (60, {"Data and API": "data.md"}, "Data"),
             (64, {"Data model": "wiki/data-model.md"}, "Data"),
             (68, {"Updated platform files": "exports.md"}, "Data"),
+            (69, {"Measurements": sitemeasurements.EXPLORER_PAGE}, "Data"),
         ],
         "contribute": [(0, "contributing.md")],
     }
@@ -11002,6 +11007,11 @@ def main() -> None:
         help="directory the pages are written to (default: the project's "
              "content collection, src/content/docs)",
     )
+    parser.add_argument(
+        "--require-measurements", action="store_true",
+        help="fail when the measurements clone does not hold the pinned commit "
+             "(the deploy build passes it; without it the site lists no measurements)",
+    )
     args = parser.parse_args()
     if args.docs_dir is None:
         args.docs_dir = str(Path(args.site_dir) / STARLIGHT_CONTENT)
@@ -11158,12 +11168,27 @@ def _generate(args: argparse.Namespace) -> None:
     if clash:
         raise RuntimeError(f"a console page takes a manufacturer's address: {sorted(clash)}")
 
+    print("Compiling the measurements at their pin...")
+    measured = sync_measurements.build(
+        public, emulators_dir=Path(args.emulators_dir), require=args.require_measurements,
+        system_names={sys_id: system_name(sys_id) for sys_id in sorted(set(_system_page_map))},
+    )
+    for path in measured.paths:
+        _record(path)
+    _MEASUREMENT_ITEMS.clear()
+    _MEASUREMENT_ITEMS.update(sitemeasurements.by_emulator(measured.index))
+    print(f"  {measured.index['records']} records, {measured.index['count']} tests")
+    write_if_changed(str(docs / sitemeasurements.EXPLORER_PAGE),
+                     sitemeasurements.generate_explorer_page(measured.index) + reference_next())
+
     print("Generating static API and bulk data exports...")
     exports = generate_data_exports(
         public, db, coverages, profiles, stats, gap_report, consumers
     )
     write_release_advertised(Path(args.site_dir) / "src" / "generated")
-    write_if_changed(str(docs / "data.md"), generate_data_page(exports) + reference_next())
+    write_if_changed(str(docs / "data.md"),
+                     generate_data_page(exports, public / sync_measurements.SQLITE_PATH)
+                     + reference_next())
     PAGE_DATA["data.md"] = dataset_structured_data(exports, db)
 
     export_reports = load_export_reports()
@@ -11319,6 +11344,10 @@ def _generate(args: argparse.Namespace) -> None:
         for src_file in wiki_src.glob("*.md"):
             shutil.copy2(src_file, wiki_dest / src_file.name)
             _record(wiki_dest / src_file.name)
+    method_page = docs / sitemeasurements.METHOD_PAGE
+    if method_page.is_file():
+        write_if_changed(str(method_page), sitemeasurements.with_hosts(
+            method_page.read_text(encoding="utf-8"), measured.index))
     table_page = docs / SETUP_TABLE_PAGE
     if table_page.is_file():
         write_if_changed(str(table_page), with_setup_table(

scripts/sitemeasurements.py

@@ -0,0 +1,173 @@
+"""The site's measurement pages, from the index sync_measurements.py writes.
+
+Three places read the same index: the Tested block of each emulator page,
+the Measurements explorer page and the host table of the methodology page.
+A number is never shown without its method and host beside it.
+"""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+from pathlib import Path
+
+import sitemarkup as markup
+from siterender import _escape_tags
+
+EXPLORER_PAGE = "measurements.md"
+METHOD_PAGE = "wiki/measurements.md"
+HOSTS_MARKER = "<!-- measurements-hosts -->"
+
+
+def by_emulator(index: dict) -> dict[str, list[dict]]:
+    out: dict[str, list[dict]] = {}
+    for item in index.get("items", []):
+        out.setdefault(item["emulator"], []).append(item)
+    return out
+
+
+def _text(value: object) -> str:
+    """Record text as the page shows it. A semicolon joining two clauses
+    ends a sentence on the site: the record keeps the words as written."""
+    text = " ".join(str(value or "").split())
+    text = re.sub(r";\s+(\w)", lambda m: ". " + m[1].upper(), text)
+    return _escape_tags(text)
+
+
+def _short(commit: str | None) -> str:
+    return f"`{commit[:12]}`" if commit else "unknown"
+
+
+def tested_item(item: dict) -> list[str]:
+    """One test: what ran, the result, the method and host beside it, the record."""
+    title = item["claim"] or f"Benchmark of {item['content'] or 'the core alone'}"
+    runs = "1 run" if item["runs"] == 1 else f"{item['runs']} runs, the latest shown"
+    core = item["core"]
+    build = f"{_text(core['library'])} at {_short(core['commit'])}" if core.get("commit") \
+        else _text(core["library"])
+    lines = [
+        f"- **{_text(title)}**",
+        "",
+        f"    **Result:** `{item['status']}`. {_text(item['statement'])}.",
+        "",
+    ]
+    if item.get("method"):
+        lines += [f"    **Method:** {_text(item['method'])}", ""]
+    lines += [
+        f"    **Ran on:** {_text(item['host'])}, {item['date'][:10]}. Core build: {build}. "
+        f"{item['variants']} variant{'s' if item['variants'] != 1 else ''}, "
+        f"{item['repetitions']} repetition{'s' if item['repetitions'] != 1 else ''} each"
+        + (f", {item['flags']} flag{'s' if item['flags'] != 1 else ''} raised" if item["flags"] else "")
+        + ".",
+        "",
+        f"    [Full record]({item['record_url']}), [its data]({item['data_url']}), {runs}.",
+        "",
+    ]
+    return lines
+
+
+def tested_section(name: str, items: list[dict]) -> list[str]:
+    """The Tested block of an emulator page, empty when nothing was recorded."""
+    if not items:
+        return []
+    lines = [
+        "## Tested",
+        "",
+        f"This emulator was run in real conditions by the [core probe](../{METHOD_PAGE}). "
+        f"Each test below is the latest of its kind. "
+        f"[All measurements](../{EXPLORER_PAGE}), "
+        f"[these records as SQLite](../downloads/measurements/{name}.sqlite).",
+        "",
+    ]
+    for item in sorted(items, key=lambda i: (i["date"], i["subject"]), reverse=True):
+        lines += tested_item(item)
+    return lines
+
+
+def hosts_table(index: dict) -> list[str]:
+    hosts: dict[str, int] = {}
+    for item in index.get("items", []):
+        # The load average differs per run: the computer is what comes before it.
+        computer = item["host"].split(", load ")[0]
+        hosts[computer] = hosts.get(computer, 0) + 1
+    if not hosts:
+        return ["No result is recorded at the pinned commit yet.", ""]
+    return [
+        "| Computer | Tests |",
+        "|----------|------:|",
+        *(f"| {_text(host)} | {count} |" for host, count in sorted(hosts.items())),
+        "",
+    ]
+
+
+def with_hosts(page: str, index: dict) -> str:
+    """The methodology page with the computers the results ran on."""
+    return page.replace(HOSTS_MARKER, "\n".join(hosts_table(index)).rstrip("\n"))
+
+
+def generate_explorer_page(index: dict) -> str:
+    commit = index.get("commit")
+    repo = index.get("repo") or ""
+    source = (f"[retrobios-measurements]({repo}/tree/{commit}) at {_short(commit)}"
+              if commit else "the measurements project, not read in this build")
+    return "\n".join([
+        "# Measurements",
+        "",
+        f"Emulators run in real conditions: {index.get('records', 0)} records, "
+        f"{index.get('count', 0)} tests, read from {source}. "
+        f"[How they are made]({METHOD_PAGE}).",
+        "",
+        " ".join([
+            markup.button("SQLite of every record", "downloads/measurements.sqlite", primary=True),
+            markup.button("Index as JSON", "data/measurements/index.json"),
+        ]),
+        "",
+        "Each row is the latest record of one test, with its method and the computer it "
+        "ran on. One SQLite file per emulator is linked from its page.",
+        "",
+        "<MeasurementsExplorer />",
+        "",
+    ])
+
+
+def sqlite_tables(path: Path) -> list[tuple[str, list[str]]]:
+    """Each table of a database with its columns, in the order created."""
+    connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
+    try:
+        names = [row[0] for row in connection.execute(
+            "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")]
+        return [(name, [row[1] for row in connection.execute(f"PRAGMA table_info({name})")])
+                for name in names]
+    finally:
+        connection.close()
+
+
+def data_page_section(path: Path) -> list[str]:
+    """The Data page's description of measurements.sqlite."""
+    return [
+        "## Measurements",
+        "",
+        "[`measurements.sqlite`](downloads/measurements.sqlite) holds every core probe "
+        "record at the pinned commit of [retrobios-measurements]"
+        "(https://github.com/Abdess/retrobios-measurements), and "
+        "`downloads/measurements/<emulator>.sqlite` the records of one emulator, linked "
+        f"from its page. The [Measurements]({EXPLORER_PAGE}) page reads "
+        "[`data/measurements/index.json`](data/measurements/index.json). A record line is "
+        "valid against [`core-probe-record.schema.json`](api/v1/schemas/core-probe-record.schema.json) "
+        "and its data file against [`core-probe-data.schema.json`](api/v1/schemas/core-probe-data.schema.json).",
+        "",
+        "| Table | Columns |",
+        "|-------|---------|",
+        *(f"| `{name}` | {', '.join(f'`{c}`' for c in columns)} |" for name, columns in sqlite_tables(path)),
+        "",
+        "`meta` names the measurements commit and the record version. A list or object "
+        "value is JSON text, an absent one empty. `runs.superseded` is 1 for a record a "
+        "later one corrects.",
+        "",
+        "```sql",
+        "SELECT r.emulator, c.claim, c.status, r.recorded_at, h.cpu_model",
+        "FROM runs r JOIN checks c ON c.run_id = r.id JOIN hosts h ON h.id = r.host_id",
+        "WHERE r.superseded = 0 ORDER BY r.recorded_at;",
+        "```",
+        "",
+    ]

scripts/sitestarlight.py

@@ -28,6 +28,7 @@ COMPONENTS = {
     "Tabs": "@astrojs/starlight/components",
     "TabItem": "@astrojs/starlight/components",
     "FilePicker": "components/FilePicker.astro",
+    "MeasurementsExplorer": "components/MeasurementsExplorer.astro",
 }
 
 # Pages of file cards: level 3 headings by the hundred, so the table of