chore: merge generated page refresh

CONTRIBUTING.md

@@ -68,6 +68,7 @@ python scripts/validate_pr.py --changed          # the BIOS files the branch add
 python scripts/validate_schemas.py --source-only # profiles and platform files
 python scripts/profile_sync.py --changed --strict  # each profile the branch adds or edits
 python scripts/refresh_data_dirs.py  # data directories the pack-building tests read
+python scripts/generate_site.py       # the pages the site tests read
 python -m unittest discover -s tests -t .
 ```
 

scripts/generate_readme.py

@@ -975,6 +975,7 @@ python scripts/validate_pr.py --changed          # the BIOS files the branch add
 python scripts/validate_schemas.py --source-only # profiles and platform files
 python scripts/profile_sync.py --changed --strict  # each profile the branch adds or edits
 python scripts/refresh_data_dirs.py  # data directories the pack-building tests read
+python scripts/generate_site.py       # the pages the site tests read
 python -m unittest discover -s tests -t .
 ```
 

scripts/generate_site.py

@@ -5295,12 +5295,19 @@ def generate_provenance_page(
                 kept.append(entry)
         data["missing"] = kept
     total_missing = distinct_missing(report)
+    # The three counts below added up to more than the total above it.
+    listed = sum(len(data["missing"]) for data in report.values())
+    overlap = (
+        f" A dump two catalogs describe is listed under each, so the lists "
+        f"hold {listed:,} rows."
+        if listed != total_missing else ""
+    )
     lines.extend(
         [
             "## Wanted: catalogued dumps this project does not have",
             "",
             f"The tables below list **{total_missing:,}** dumps a catalog "
-            "describes and the collection lacks. Nothing here ships in any "
+            f"describes and the collection lacks.{overlap} Nothing here ships in any "
             "pack: it is an acquisition list, published with hashes so anyone "
             "can check a personal collection against it. A contribution "
             "matching one of these hashes is welcome: see "
@@ -10060,7 +10067,8 @@ def _copy_cell(path: str, algos: list[str], db: dict | None) -> str:
 
 
 def _render_problem_files(
-    platform_problems: list, settled: list | None = None, db: dict | None = None
+    platform_problems: list, settled: list | None = None, db: dict | None = None,
+    missing_total: int | None = None,
 ) -> list[str]:
     """Files a platform declares that its own check then rejects.
 
@@ -10106,12 +10114,17 @@ def _render_problem_files(
             entry["platforms"].append(p["platform"])
 
     total_problems = len(missing_files) + len(untested_files) + len(mismatch_files)
+    # One row per name: the page opened on "48 are missing", counted by
+    # destination, above a heading that said "10 files". Both are said.
+    names = _count(len(missing_files), "name")
+    missing_said = (f"{missing_total:,} missing under {names}"
+                    if missing_total is not None else f"{len(missing_files)} missing")
 
     if total_problems > 0:
         lines.extend([
             "## Problem files",
             "",
-            f"{len(missing_files)} missing, {len(untested_files)} untested, "
+            f"{missing_said}, {len(untested_files)} untested, "
             f"{len(mismatch_files)} discrepancy.",
             "",
         ])
@@ -10119,13 +10132,14 @@ def _render_problem_files(
         if missing_files:
             lines.extend([
                 f'### Missing <span class="rb-badge rb-badge-danger">'
-                f"{_count(len(missing_files), 'file')}</span>",
+                f"{_count(len(missing_files), 'name')}</span>",
                 "",
                 "Required is the platform's own flag. Category says what the "
                 "platform's emulators do with the file: `bios` is firmware they "
                 "load, `game_data` comes from a copy of the game, `unread` is "
                 "opened by none of them. Only a missing `bios` entry stops a "
-                "console from starting.",
+                "console from starting. A name a list declares in several "
+                "folders is one row.",
                 "",
                 "| File | Category | Required | Platforms |",
                 "|------|----------|----------|-----------|",
@@ -10440,9 +10454,9 @@ def generate_gap_analysis(
             prof_cell = gt_cell = "-"
         elif gt["total"]:
             prof_pct = f"{gt.get('with_profile', 0) / gt['total'] * 100:.0f}%"
-            prof_cell = f"{gt.get('with_profile', 0)} ({prof_pct})"
+            prof_cell = f"{gt.get('with_profile', 0):,} ({prof_pct})"
             gt_pct = f"{gt['with_validation'] / gt['total'] * 100:.0f}%"
-            gt_cell = f"{gt['with_validation']} ({gt_pct})"
+            gt_cell = f"{gt['with_validation']:,} ({gt_pct})"
         else:
             prof_cell = gt_cell = "0"
         lines.append(
@@ -10513,14 +10527,14 @@ def generate_gap_analysis(
         "so there is nothing to compare: the platform's own source is the "
         "only authority for its files.",
         "",
-        "The same comparison drives `scripts/exporter/`: each platform's "
-        "own file can be regenerated in its native format with these "
-        "differences applied (System.dat, es_bios.xml, "
-        "batocera-systems.json, ...).",
+        "The same comparison feeds the [updated platform files](exports.md): "
+        "each platform's own file, patched with these differences and kept "
+        "in its native format (System.dat, es_bios.xml, batocera-systems, ...).",
         "",
     ])
 
-    lines.extend(_render_problem_files(platform_problems, settled_problems, db))
+    lines.extend(_render_problem_files(platform_problems, settled_problems, db,
+                                       total_missing_verify))
 
     # ---- Section 3: Core complement (cross-reference provenance) ----
 
@@ -10572,8 +10586,8 @@ def generate_gap_analysis(
             f"{src_totals['missing']:,} to source.",
             "",
             "This counts every profiled emulator, including those no platform "
-            "ships yet, so it reaches past what the pack tables above measure: "
-            "those cover only the emulators each platform actually ships. "
+            "ships yet, so it reaches past what the first table above counts: "
+            "it covers only the emulators each platform actually ships. "
             "Whatever is not in the collection is an acquisition target, named "
             "per emulator below.",
             "",
@@ -11027,6 +11041,12 @@ def generate_cross_reference(
     total_upstreams = len({
         upstream_project(_xref_upstream(p))[0] for p in unique.values() if _xref_upstream(p)
     })
+    # "623 cores across 18 platforms" counted every profile: 91 of them
+    # no platform runs.
+    run_cores = len({
+        key for cov in coverages.values()
+        for key in resolve_platform_cores(cov["config"], unique)
+    })
 
     lines = [
         "# Cores by platform",
@@ -11035,8 +11055,9 @@ def generate_cross_reference(
         "it comes from. A core built for RetroArch is often a port of another "
         "emulator, and the two can differ in the files they read and the checks "
         "they make, so each core names its upstream project. "
-        f"{total_cores} cores across {len(coverages)} platforms come from "
-        f"{total_upstreams} upstream projects.",
+        f"{total_cores} cores are profiled, {run_cores} of them run by the "
+        f"{len(coverages)} platforms, and they come from {total_upstreams} "
+        "upstream projects.",
         "",
         "The files a platform's emulators load and its own list does not name are "
         "counted on [What the packs do not hold](gaps.md).",

tests/test_audit_regressions.py

@@ -5002,6 +5002,19 @@ class GapsPageCountsOneWay(unittest.TestCase):
         self.assertIn("Other revisions of a held destination", text)
         self.assertIn("`cdibios.zip`", text)
 
+    def test_missing_says_the_destination_count_of_the_opening_sentence(self):
+        # The page opened on "48 are missing" (by destination) above a
+        # heading that said "10 files": one row per name.
+        from scripts.generate_site import _render_problem_files
+
+        problems = [{"platform": "AmberELEC", "name": "hardwareconfig.xml",
+                     "status": "missing", "required": True, "reason": "",
+                     "discrepancy": "", "category": "unread"}] * 3
+        text = "\n".join(_render_problem_files(problems, missing_total=3))
+        self.assertIn("3 missing under 1 name,", text)
+        self.assertIn('rb-badge-danger">1 name</span>', text)
+        self.assertNotIn("1 missing", text)
+
     def test_verify_marks_the_settled_entry(self):
         sys.path.insert(0, str(ROOT / "scripts"))
         import verify

tests/test_doc_surfaces.py

@@ -9,10 +9,12 @@ tool prints, and a size is the one the release record holds.
 
 from __future__ import annotations
 
+import ast
 import contextlib
 import re
 import sys
 import unittest
+from functools import cache
 from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
@@ -380,6 +382,24 @@ class ContributorChecksMatchCi(unittest.TestCase):
             workflow.index("python -m unittest discover"),
         )
 
+    def test_the_pr_checklist_generates_the_site_before_the_tests(self):
+        # validate.yml generates the pages before the suite. Without them
+        # the 228 tests that read the gitignored pages fail on a clone.
+        workflow = _read(".github/workflows/validate.yml")
+        self.assertLess(
+            workflow.index("run: python scripts/generate_site.py"),
+            workflow.index("python -m unittest discover"),
+        )
+        for text in (
+            contributing_text(lambda page: page, "# C", "here"),
+            _read("CONTRIBUTING.md"),
+        ):
+            block = _section(text, "## Before opening a PR")
+            self.assertIn("python scripts/generate_site.py", block)
+            self.assertLess(
+                block.index("generate_site.py"), block.index("unittest discover")
+            )
+
 
 class PagesOpenOnTheirContent(unittest.TestCase):
     """The download page said "This page picks the right pack for a setup"
@@ -396,5 +416,180 @@ class PagesOpenOnTheirContent(unittest.TestCase):
         self.assertEqual(announcing, [])
 
 
+
+
+class CoreCountsReadFromTheData(unittest.TestCase):
+    """The cores page said "623 cores across 18 platforms": that was every
+    profile, and 91 of them no platform runs."""
+
+    def test_the_run_count_is_what_the_platforms_resolve(self):
+        import yaml
+        from common import load_emulator_profiles, load_platform_config, resolve_platform_cores
+
+        from tests import sitepages
+
+        sitepages.require()
+        text = sitepages.page("cross-reference.md").read_text(encoding="utf-8")
+        profiles = load_emulator_profiles(str(REPO_ROOT / "emulators"))
+        unique = {k: v for k, v in profiles.items() if v.get("type") not in ("alias", "test")}
+        registry = yaml.safe_load((REPO_ROOT / "platforms" / "_registry.yml").read_text(
+            encoding="utf-8"))["platforms"]
+        run = set()
+        for name in registry:
+            config = load_platform_config(name, str(REPO_ROOT / "platforms"))
+            run |= set(resolve_platform_cores(config, unique))
+        said = re.search(r"([\d,]+) cores are profiled, ([\d,]+) of them run by the (\d+) platforms",
+                         text)
+        self.assertIsNotNone(said, "the page says how many profiled cores the platforms run")
+        self.assertEqual(int(said[1].replace(",", "")), len(unique))
+        self.assertEqual(int(said[2].replace(",", "")), len(run))
+        self.assertEqual(int(said[3]), len(registry))
+
+
+# What the generators write must not bring back what the project retired.
+# The generated pages, README.md, CONTRIBUTING.md and the site's components
+# are read for a retired command, address or feature term, each listed with
+# the reason it was retired, and for every `--option` a generated command
+# passes: the script it names must declare it in its parser. The pages copied
+# from wiki/ are left to their own tests: the installer page keeps the raw
+# one-liner as the documented fallback.
+
+
+def _sitepages():
+    from tests import sitepages
+
+    return sitepages
+
+# (string, why it is retired). Compared case-sensitively against the text.
+RETIRED = (
+    ("raw.githubusercontent.com/Abdess/retrobios/main/install.sh",
+     "the site serves the one-liner at https://abdess.github.io/retrobios/i "
+     "since 2026-10-28, pinned to the commit the site was built from"),
+    ("raw.githubusercontent.com/Abdess/retrobios/main/install.ps1",
+     "served at https://abdess.github.io/retrobios/i.ps1 since 2026-10-28"),
+    ("Abdess/retroarch_system",
+     "the repository was renamed retrobios in March 2026"),
+    ("mkdocs serve", "the MkDocs site was replaced by Astro Starlight on 2026-10-22"),
+    ("mkdocs build", "the MkDocs site was replaced by Astro Starlight on 2026-10-22"),
+    ("pip install mkdocs", "the site is built by Node from website/package-lock.json"),
+    ('class="rb-tested', "the Tested blocks were removed on 2026-10-28: "
+     "only Proof lines and the Measurements page remain"),
+    ("rb-region-runs", "region outcomes left the system pages on 2026-10-28"),
+    ("## Campaign", "the campaign table left the Measurements page on 2026-10-28"),
+    ("| Tested by |", "the coverage table left the Measurements page on 2026-10-28"),
+    ("ms per frame", "benchmark figures stay in the records and SQLite files, "
+     "never on a page, since 2026-10-28"),
+    ("with this BIOS folder as", "the Tested block of platform pages, removed on 2026-10-28"),
+    ("regenerated in its native format", "the export patches each platform's own "
+     "file and never regenerates it (scripts/exporter/base_exporter.py needs_original)"),
+)
+
+COMMAND = re.compile(
+    r"(?:python3? (?P<script>scripts/[\w/]+\.py|install\.py)"
+    r"|python3? -m (?P<module>scripts(?:\.\w+)+)"
+    r"|(?P<oneliner>retrobios/i \| sh -s --|retrobios/i\.ps1\)\)\)))"
+    r"(?P<args>[^\n`<|]*)"
+)
+OPTION = re.compile(r"(?<![\w-])(--[a-z][a-z0-9-]*)")
+
+
+def retired_hits(text: str) -> list[str]:
+    return [term for term, _why in RETIRED if term in text]
+
+
+@cache
+def parser_options(script: str) -> frozenset[str]:
+    """The long options *script* declares through add_argument, with the
+    shared ones of scraper_cli for a scraper that hands its parser over."""
+    source = (REPO_ROOT / script).read_text(encoding="utf-8")
+    tree = ast.parse(source)
+    found = set()
+    for node in ast.walk(tree):
+        if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)
+                and node.func.attr == "add_argument"):
+            found.update(arg.value for arg in node.args
+                         if isinstance(arg, ast.Constant) and isinstance(arg.value, str)
+                         and arg.value.startswith("--"))
+    if "scraper_cli(" in source and script != "scripts/scraper/base_scraper.py":
+        found |= parser_options("scripts/scraper/base_scraper.py")
+    return frozenset(found)
+
+
+def unknown_options(text: str) -> list[str]:
+    """Each `script --option` of a command whose script does not declare it."""
+    wrong = []
+    for match in COMMAND.finditer(text):
+        if match["oneliner"]:
+            script = "install.py"
+        elif match["module"]:
+            script = match["module"].replace(".", "/") + ".py"
+        else:
+            script = match["script"]
+        if not (REPO_ROOT / script).is_file():
+            wrong.append(f"{script}: no such script")
+            continue
+        known = parser_options(script)
+        wrong += [f"{script} {option}" for option in OPTION.findall(match["args"])
+                  if option not in known]
+    return wrong
+
+
+def _surfaces() -> dict[str, str]:
+    content = _sitepages().require()
+    texts = {
+        str(path.relative_to(REPO_ROOT)): path.read_text(encoding="utf-8")
+        for path in sorted(content.rglob("*.md*"))
+        if "wiki" not in path.relative_to(content).parts
+    }
+    for name in ("README.md", "CONTRIBUTING.md"):
+        texts[name] = (REPO_ROOT / name).read_text(encoding="utf-8")
+    for path in sorted((REPO_ROOT / "website" / "src").glob("*/*")):
+        if path.suffix in (".astro", ".ts", ".css", ".mjs"):
+            texts[str(path.relative_to(REPO_ROOT))] = path.read_text(encoding="utf-8")
+    return texts
+
+
+class RetiredTermsStayRetired(unittest.TestCase):
+    def test_each_retired_term_says_why(self):
+        for term, why in RETIRED:
+            with self.subTest(term=term):
+                self.assertTrue(term.strip())
+                self.assertGreater(len(why.split()), 4)
+
+    def test_the_check_finds_what_the_old_output_wrote(self):
+        old = (
+            "curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh\n"
+            '## Tested\n\n<div class="rb-tested">\n'
+            "The same comparison drives `scripts/exporter/`: each platform's own "
+            "file can be regenerated in its native format with these differences.\n"
+        )
+        self.assertEqual(retired_hits(old), [
+            "raw.githubusercontent.com/Abdess/retrobios/main/install.sh",
+            'class="rb-tested', "regenerated in its native format",
+        ])
+
+    def test_no_surface_brings_one_back(self):
+        found = {name: hits for name, text in _surfaces().items()
+                 if (hits := retired_hits(text))}
+        self.assertEqual(found, {})
+
+
+class EveryOptionExists(unittest.TestCase):
+    def test_the_check_reads_each_form_of_command(self):
+        text = (
+            "python scripts/generate_pack.py --platform retroarch --zip-all\n"
+            "curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --check --frobnicate\n"
+            "&([scriptblock]::Create((irm https://abdess.github.io/retrobios/i.ps1))) --dry-run\n"
+            "python -m scripts.scraper.libretro_scraper --dry-run\n"
+        )
+        self.assertEqual(unknown_options(text), [
+            "scripts/generate_pack.py --zip-all", "install.py --frobnicate",
+        ])
+
+    def test_every_generated_command_names_options_its_script_has(self):
+        wrong = {name: bad for name, text in _surfaces().items()
+                 if (bad := unknown_options(text))}
+        self.assertEqual(wrong, {})
+
 if __name__ == "__main__":
     unittest.main()

tests/test_site_r3_css.py

@@ -11,7 +11,9 @@ import unittest
 
 from tests.test_site_r2_css import CSS_PATH, PAGE_BG, _blocks, _ratio, _rules, _strip_comments, _theme_tokens
 
-PACKS = "table.rb-packs"
+# The Get your files tables: no class reaches them, the Markdown table is
+# matched by the cells the generator labels.
+PACKS = "table:has(td[data-label='Runs on']):has(td[data-label='ZIP'])"
 
 
 class ThemeStylesheetRound3(unittest.TestCase):

website/src/styles/retrobios.css

@@ -371,19 +371,19 @@
 	.sl-markdown-content :is(td.rb-reason, td[data-label='Reason'], td:has(> .rb-reason)) { min-width: 12rem; }
 
 	/* Get your files: every group table shares one set of columns. */
-	.sl-markdown-content :is(table.rb-packs, table:has(td[data-label='Runs on']):has(td[data-label='ZIP'])) {
+	.sl-markdown-content table:has(td[data-label='Runs on']):has(td[data-label='ZIP']) {
 		display: table;
 		table-layout: fixed;
 		width: 100%;
 	}
-	.sl-markdown-content :is(table.rb-packs, table:has(td[data-label='Runs on']):has(td[data-label='ZIP'])) th:nth-child(1) { width: 18%; }
-	.sl-markdown-content :is(table.rb-packs, table:has(td[data-label='Runs on']):has(td[data-label='ZIP'])) th:nth-child(2) { width: 20%; }
-	.sl-markdown-content :is(table.rb-packs, table:has(td[data-label='Runs on']):has(td[data-label='ZIP'])) th:nth-child(3) { width: 40%; }
-	.sl-markdown-content :is(table.rb-packs, table:has(td[data-label='Runs on']):has(td[data-label='ZIP'])) th:nth-child(4) { width: 22%; }
-	.sl-markdown-content :is(table.rb-packs, table:has(td[data-label='Runs on']):has(td[data-label='ZIP'])) td { overflow-wrap: break-word; }
+	.sl-markdown-content table:has(td[data-label='Runs on']):has(td[data-label='ZIP']) th:nth-child(1) { width: 18%; }
+	.sl-markdown-content table:has(td[data-label='Runs on']):has(td[data-label='ZIP']) th:nth-child(2) { width: 20%; }
+	.sl-markdown-content table:has(td[data-label='Runs on']):has(td[data-label='ZIP']) th:nth-child(3) { width: 40%; }
+	.sl-markdown-content table:has(td[data-label='Runs on']):has(td[data-label='ZIP']) th:nth-child(4) { width: 22%; }
+	.sl-markdown-content table:has(td[data-label='Runs on']):has(td[data-label='ZIP']) td { overflow-wrap: break-word; }
 }
 /* A ZIP part link or button never breaks across two lines. */
-.sl-markdown-content :is(table.rb-packs, table:has(td[data-label='Runs on']):has(td[data-label='ZIP'])) td:last-child :is(a, .rb-button) { white-space: nowrap; }
+.sl-markdown-content table:has(td[data-label='Runs on']):has(td[data-label='ZIP']) td:last-child :is(a, .rb-button) { white-space: nowrap; }
 
 /* Key and value cards: the empty header row is not drawn */
 .rb-meta-card thead,
@@ -636,8 +636,8 @@ body:has(table.rb-entries) { --sl-content-width: var(--rb-grid-width); }
 .sl-markdown-content details.rb-details > summary::before { background-color: var(--rb-text-secondary); }
 .sl-markdown-content details.rb-details[open] > summary { margin-bottom: 0.6rem; }
 
-/* A platform's file list (generate_site._platform_file_list), on the
-   pattern of the Tested block: each row a details whose summary is the
+/* A platform's file list (generate_site._platform_file_list): each row
+   a details whose summary is the
    row (name, folder, status), its body the evidence across the whole
    list. The short hash sits on its copy button (site.ts) beside the
    summary, since a summary is itself a control. The details reaches under